Add automated spam filter for questions with logging according to weights

seed_spam_rules.py

@@ -0,0 +1,116 @@
+# Script to seed the database with predefined spam rules
+import os
+import django
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "forums.settings")
+django.setup()
+
+from django.db.models import Q
+from website.models import SpamRule
+
+
+
+def seed_spam_rules():
+    rules = {
+        # Certification/Exam dump patterns
+        "Certification/Exam Spam": {
+            "score": 30,
+            "type": SpamRule.KEYWORD,
+            "patterns": [
+                r"exam\s+dumps?", r"braindumps?", r"practice\s+test",
+                r"certification\s+exam", r"test\s+preparation",
+                r"exam\s+questions?", r"study\s+guides?",
+                r"pdf\s+\+\s+testing\s+engine", r"testing\s+engine",
+                r"exam\s+prep", r"mock\s+exam", r"real\s+exam",
+                r"dumps\s+pdf", r"braindump"
+            ],
+        },
+
+        # Promotional spam
+        "Promotional Spam": {
+            "score": 25,
+            "type": SpamRule.KEYWORD,
+            "patterns": [
+                r"click\s+here", r"join\s+now", r"limited\s+time",
+                r"discount", r"coupon\s+code", r"20%\s+off",
+                r"free\s+download", r"get\s+certified",
+                r"unlock\s+your\s+career", r"master\s+the",
+                r"boost\s+your\s+career", r"cert20",
+                r"at\s+checkout", r"special\s+offer",
+            ],
+        },
+
+        # Suspicious domains
+        "Suspicious Domain": {
+            "score": 35,
+            "type": SpamRule.DOMAIN,
+            "patterns": [
+                r"dumpscafe\.com", r"certsout\.com", r"mycertshub\.com",
+                r"vmexam\.com", r"kissnutra\.com", r"dumps.*\.com",
+                r"cert.*\.com", r"exam.*\.com",
+            ],
+        },
+
+        # Generic business language
+        "Business/Career Spam": {
+            "score": 15,
+            "type": SpamRule.KEYWORD,
+            "patterns": [
+                r"attests\s+to\s+your\s+proficiency",
+                r"esteemed\s+(?:accreditation|certification|credential)",
+                r"valuable\s+asset\s+to\s+companies",
+                r"demonstrates\s+your\s+ability",
+                r"comprehensive\s+study\s+(?:tools|materials)",
+                r"interactive\s+practice\s+tests",
+                r"real\s+exam\s+questions",
+                r"actual\s+exam\s+questions",
+                r"validated\s+by\s+.*certification",
+                r"urgently\s+need\s+experts",
+            ],
+        },
+
+        # Gaming content
+        "Gaming Spam": {
+            "score": 20,
+            "type": SpamRule.KEYWORD,
+            "patterns": [
+                r"spacebar\s+clicker", r"clicker\s+game",
+                r"addictive\s+game", r"upgrades\s+available",
+                r"instant\s+rewards",
+            ],
+        },
+
+        # Health/Supplement spam
+        "Health Spam": {
+            "score": 22,
+            "type": SpamRule.KEYWORD,
+            "patterns": [
+                r"vitalit[äa]t", r"nahrungserg[äa]nzungsmittel",
+                r"libido", r"fruchtbarkeit", r"energie",
+                r"hormonelle\s+balance", r"perforan",
+            ],
+        },
+    }
+
+    inserted, skipped = 0, 0
+    for note, config in rules.items():
+        for pattern in config["patterns"]:
+            exists = SpamRule.objects.filter(
+                Q(pattern=pattern) & Q(type=config["type"])
+            ).exists()
+            if not exists:
+                SpamRule.objects.create(
+                    type=config["type"],
+                    pattern=pattern,
+                    score=config["score"],
+                    notes=note,
+                )
+                inserted += 1
+            else:
+                skipped += 1
+
+    print(f"✅ Inserted {inserted} new rules, skipped {skipped} existing ones.")
+
+
+# Run it
+seed_spam_rules()

website/forms.py

@@ -69,4 +69,4 @@ def __init__(self, *args, **kwargs):
 
 class AnswerQuesitionForm(forms.Form):
     question = forms.IntegerField(widget=forms.HiddenInput())
-    body = forms.CharField(widget=forms.Textarea())
+    body = forms.CharField(widget=forms.Textarea())

website/helpers.py

@@ -1,10 +1,26 @@
 import re
-from website.models import Question
-from nltk.corpus import stopwords 
+import json
+import logging
+from datetime import datetime
+from typing import Dict, List, Tuple, Optional
+from website.models import Question, User
+from nltk.corpus import stopwords
 from nltk.tokenize import word_tokenize
+from website.templatetags.permission_tags import can_edit, can_hide_delete
 from sklearn.metrics.pairwise import cosine_similarity
+from django.conf import settings
+from django.utils import timezone
+from django.db.models import Q
+import re
+from .models import SpamRule, SpamLog   # assuming app is `forum`
+
 sw = stopwords.words('english')
 
+# Configure logging for spam detection
+logging.basicConfig(level=logging.INFO)
+spam_logger = logging.getLogger('spam_detection')
+
+
 def get_video_info(path):
     """Uses ffmpeg to determine information about a video. This has not been broadly
     tested and your milage may vary"""
@@ -67,4 +83,152 @@ def get_similar_questions(user_ques,question):
         if w in question: l2.append(1) 
         else: l2.append(0)        
     cs = cosine_similarity((l1,l2))
-    return cs[0][1]
+    return cs[0][1]
+
+
+# helpers.py
+
+MULTIPLE_URL_WEIGHT = 20
+MULTIPLE_URL_THRESHOLD = 3
+
+class SpamQuestionDetector:
+    def __init__(self):
+        # load only active + not expired rules
+        now = timezone.now()
+        qs = SpamRule.objects.filter(active=True).filter(
+            Q(expires_at__isnull=True) | Q(expires_at__gt=now)
+        )
+        self._compiled = []
+        for r in qs:
+            try:
+                cre = re.compile(r.pattern, re.IGNORECASE)
+            except re.error:
+                spam_logger.warning(f"Invalid regex in SpamRule id={r.id}: {r.pattern}")
+                continue
+            self._compiled.append({
+                'rule': r,
+                'compiled': cre
+            })
+
+    def extract_urls(self, text: str):
+        return re.findall(r'https?://[^\s)<>"]+', text)
+
+    def detect_spam(self, title: str, content: str, category: str = "", tutorial: str = "") -> dict:
+        combined_text = " ".join(filter(None, [title, content, category, tutorial])).lower()
+        spam_score = 0
+        matches = []
+
+        for entry in self._compiled:
+            rule = entry['rule']
+            cre = entry['compiled']
+            if cre.search(combined_text):
+                spam_score += rule.score
+                matches.append({
+                    'id': rule.id,
+                    'pattern': rule.pattern,
+                    'score': rule.score,
+                    'type': rule.type,
+                    'notes': rule.notes
+                })
+
+        # detect multiple URLs (we keep this behaviour from original)
+        urls = self.extract_urls(combined_text)
+        if len(urls) >= MULTIPLE_URL_THRESHOLD:
+            spam_score += MULTIPLE_URL_WEIGHT
+            matches.append({
+                'pattern': f'{len(urls)} URLs',
+                'score': MULTIPLE_URL_WEIGHT,
+                'type': 'urls'
+            })
+
+        # classification (same thresholds as earlier)
+        if spam_score >= 60:
+            confidence, action = 'HIGH', 'DELETE'
+        elif spam_score >= 30:
+            confidence, action = 'MEDIUM', 'REVIEW'
+        elif spam_score >= 15:
+            confidence, action = 'LOW', 'REVIEW'
+        else:
+            confidence, action = 'CLEAN', 'APPROVE'
+
+        result = {
+            'spam_score': spam_score,
+            'matches': matches,
+            'confidence': confidence,
+            'recommended_action': action,
+            'url_count': len(urls)
+        }
+
+        # debug log
+        spam_logger.info(f"SpamDetect result: score={spam_score} action={action} matches={len(matches)}")
+
+        return result
+
+
+def handle_spam(question, user, delete_on_high=True, save_question_metadata_before_delete=True):
+    """
+    Runs detection on a saved Question instance and logs/takes action.
+    - question: saved Question instance (has .id)
+    - user: Django user instance who created the question (for logging)
+    - delete_on_high: if True, HIGH confidence -> delete from DB; otherwise hide it (status=0)
+    Returns a status string: 'AUTO_DELETE', 'FLAGGED', 'APPROVED', 'HIDDEN'
+    """
+    detector = SpamQuestionDetector()
+    result = detector.detect_spam(
+        title=getattr(question, 'title', '') or '',
+        content=getattr(question, 'body', '') or '',
+        category=getattr(question, 'category', '') or '',
+        tutorial=getattr(question, 'tutorial', '') or ''
+    )
+
+    spam_score = result['spam_score']
+    confidence = result['confidence']
+    action = result['recommended_action']
+    details = result['matches']
+
+    # prepare log payload
+    log_payload = {
+        #'question_id': question.id,
+        'user_id': user.id if user else None,
+        'category': getattr(question, 'category', '') or '',
+        'title': getattr(question, 'title', '') or '',
+        'content': getattr(question, 'body', '') or '',
+        'action': None,
+        'spam_score': spam_score,
+        'confidence': confidence,
+        'details': details
+    }
+
+    # TAKE ACTION
+    if action == 'DELETE' and confidence == 'HIGH':
+        log_payload['action'] = 'AUTO_DELETE'
+        SpamLog.objects.create(**log_payload)
+
+        if delete_on_high:
+            # delete after logging
+            spam_logger.info(f"AUTO_DELETE: Question {question.id} by user {user.id} score={spam_score}")
+            question.delete()
+            return 'AUTO_DELETE'
+        else:
+            # hide instead of delete
+            question.spam = True
+            question.status = 0
+            question.save(update_fields=['spam', 'status'])
+            return 'HIDDEN'
+
+    elif action == 'REVIEW':
+        # flag for admin review
+        log_payload['action'] = 'FLAGGED'
+        SpamLog.objects.create(**log_payload)
+
+        question.approval_required = True
+        question.spam = False
+        question.save(update_fields=['approval_required', 'spam'])
+        return 'FLAGGED'
+
+    else:
+        # APPROVE / CLEAN
+        question.spam = False
+        question.approval_required = False
+        question.save(update_fields=['spam', 'approval_required'])
+        return 'APPROVED'

website/models.py

@@ -1,6 +1,6 @@
 from django.db import models
 from django.contrib.auth import get_user_model
-
+import json
 User = get_user_model()
 
 
@@ -18,6 +18,11 @@ class Question(models.Model):
     status = models.IntegerField(default=1)
     last_active = models.DateTimeField(null=True)
     last_post_by = models.IntegerField(null=True)
+    spam = models.BooleanField(default=False)
+    approval_required = models.BooleanField(default=False)
+    approved_by = models.IntegerField(null=True)
+    date_approved = models.DateTimeField(auto_now_add=True)
+
     # votes = models.IntegerField(default=0)
 
     def user(self):
@@ -88,3 +93,49 @@ def poster(self):
         return user.username
 
 # CDEEP database created using inspectdb arg of manage.py
+class SpamRule(models.Model):
+    KEYWORD = "keyword"
+    DOMAIN = "domain"
+    TYPES = [(KEYWORD, "Keyword"), (DOMAIN, "Domain / URL")]
+
+    type = models.CharField(max_length=10, choices=TYPES)
+    pattern = models.CharField(max_length=500)
+    score = models.IntegerField(default=1)
+    active = models.BooleanField(default=True)
+    notes = models.CharField(max_length=200, blank=True)
+    expires_at = models.DateTimeField(null=True, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    def __str__(self):
+        return f"{self.type}: {self.pattern} ({self.score})"
+
+class SpamLog(models.Model):
+    ACTIONS = [
+        ("AUTO_DELETE", "Auto Deleted"),
+        ("FLAGGED", "Flagged for Review"),
+        ("APPROVED", "Approved"),
+    ]
+
+    #question_id = models.IntegerField()
+    user_id = models.IntegerField(null=True, blank=True)
+    category = models.CharField(max_length=200, blank=True)
+    title = models.CharField(max_length=200, blank=True)
+    content = models.TextField(blank=True)
+    action = models.CharField(max_length=20, choices=ACTIONS)
+    spam_score = models.IntegerField()
+    confidence = models.CharField(max_length=20)
+    details = models.TextField(blank=True, null=True)
+
+    def set_details(self, data):
+        self.details = json.dumps(data)
+
+    def get_details(self):
+        try:
+            return json.loads(self.details)
+        except Exception:
+            return {}
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return f"Q{self.question_id} - {self.action} ({self.spam_score})"