similar question (#31)

* similar question

* requirements added

* ordering improved

* ajax typing

Co-authored-by: root <root@localhost.localdomain>

Author: MetalBlazer

requirements.txt

@@ -12,4 +12,6 @@ django-compressor==2.2
 django-extensions==2.1.3
 django-filter==2.0.0
 django-debug-toolbar==1.4
-python-dotenv==0.10.3
+python-dotenv==0.10.3
+nltk==3.5
+sklearn==0.0

static/website/js/custom.js

@@ -3,6 +3,7 @@ $(document).ready(function() {
     $tutorial = $("#id_tutorial");
     $minute_range = $("#id_minute_range");
     $second_range = $("#id_second_range");
+    $title = $("#id_title");
 		var tutorial = $tutorial.val();
 		var category = $category.val();
 
@@ -98,15 +99,16 @@ $(document).ready(function() {
         }
     });
 
-    $second_range.change(function() {
+    $title.keyup(function() {
         $.ajax({
             url: "/ajax-similar-questions/",
             type: "POST",
             data: {
                 category: $category.val(),
                 tutorial: $tutorial.val(),
                 minute_range: $minute_range.val(),
-                second_range: $second_range.val()
+                second_range: $second_range.val(),
+                title: $title.val()
             },
             dataType: "html",
             success: function(data) {

static/website/templates/ajax-similar-questions.html

@@ -1,10 +1,10 @@
 <div>
 <div id="similar-count">
-    {% if questions.count > 0 %}
-        {% if questions.count == 1 %}
-            1 similar question
+    {% if questions_count > 0 %}
+        {% if questions_count == 1 %}
+            1 similar, previously asked question on our Forums
         {% else %}
-            {{ questions.count }} similar questions
+            {{ questions_count }} similar, previously asked questions on our Forums
         {% endif %}
     {% endif %}
 </div>

static/website/templates/new-question.html

@@ -31,13 +31,6 @@ <h4>
         <div class="col-lg-2 col-md-2 col-sm-2">
             {% render_field form.second_range class+="form-control" disabled="disabled" %} 
         </div>
-        <div class="col-lg-2 col-md-2 col-sm-2">
-            <small><strong>
-            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
-                0 similar questions
-            </a>
-            </strong></small>
-        </div>
     </div>
     <hr>
 
@@ -48,6 +41,15 @@ <h4>
                 <label for="id_title">Title:</label>
                 {% render_field form.title class+="form-control" %} 
             </div>
+
+            <div class="form=group">
+            <small><strong>
+            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
+                0 similar, previously asked question on our Forums
+            </a>
+            </strong></small>
+        </div>
+
             <div class="form-group">
                 <label for="id_body">Question:</label>
                 {% render_field form.body class+="form-control" %}

website/helpers.py

@@ -1,5 +1,9 @@
 import re
-
+from website.models import Question
+from nltk.corpus import stopwords 
+from nltk.tokenize import word_tokenize
+from sklearn.metrics.pairwise import cosine_similarity
+sw = stopwords.words('english')
 
 def get_video_info(path):
     """Uses ffmpeg to determine information about a video. This has not been broadly
@@ -37,3 +41,30 @@ def prettify(string):
     string = re.sub('[^A-Za-z0-9\-]+', '', string)
     string = re.sub('-+', '-', string)
     return string
+
+
+def pre_process(text):
+    text=text.lower()                       # lowercase
+    text=re.sub("<!--?.*?-->","",text)      # remove tags
+    text=re.sub("(\\d|\\W)+"," ",text)      # remove special characters and digits
+    return text
+
+def clean_user_data(text):
+    words = word_tokenize(pre_process(text.lower()))
+    clean_list = [w for w in words if not w in sw]
+    return clean_list
+
+def get_similar_questions(user_ques,question):
+    total = []
+    l1 = []
+    l2 = []
+    question = word_tokenize(pre_process(question))
+    question =[w for w in question if not w in sw]
+    total = user_ques + question
+    for w in total: 
+        if w in user_ques: l1.append(1) # create a vector 
+        else: l1.append(0) 
+        if w in question: l2.append(1) 
+        else: l2.append(0)        
+    cs = cosine_similarity((l1,l2))
+    return cs[0][1]

website/views.py

@@ -12,7 +12,7 @@
 from website.models import Question, Answer, Notification, AnswerComment
 from spoken_auth.models import TutorialDetails, TutorialResources
 from website.forms import NewQuestionForm, AnswerQuesitionForm
-from website.helpers import get_video_info, prettify
+from website.helpers import get_video_info, prettify, clean_user_data, get_similar_questions
 from django.conf  import settings
 from website.templatetags.permission_tags import can_edit
 from spoken_auth.models import FossCategory
@@ -546,15 +546,22 @@ def ajax_answer_comment_update(request):
 
 def ajax_similar_questions(request):
     if request.method == 'POST':
-        category = request.POST['category']
-        tutorial = request.POST['tutorial']
-        # minute_range = request.POST['minute_range']
-        # second_range = request.POST['second_range']
-
-        # add more filtering when the forum grows
-        questions = Question.objects.filter(category=category).filter(tutorial=tutorial)
+        category = request.POST['category'].replace(' ','-')
+        tutorial = request.POST['tutorial'].replace(' ','-')
+        title = request.POST['title']
+        user_title = clean_user_data(title)
+        # Increase the threshold as the Forums questions increase
+        THRESHOLD = 0.3
+        top_ques = []
+        questions = Question.objects.filter(category=category,tutorial=tutorial)
+        for question in questions:
+             question.similarity= get_similar_questions(user_title,question.title)
+             if question.similarity >= THRESHOLD:
+                top_ques.append(question)
+        top_ques = sorted(top_ques,key=lambda x : x.similarity, reverse=True)
         context = {
-            'questions': questions
+            'questions': top_ques,
+            'questions_count':len(top_ques)
         }
         return render(request, 'website/templates/ajax-similar-questions.html', context)