Merge pull request ninenine#29 from pitaj/master

Unicode support, settings updated correctly

Author: ninenine

index.js

@@ -24,7 +24,7 @@ var Beep = {
 
 	parseContent: function (content) {
 		var nil = '^(?!x)x';
-		return parseContent(content, Beep.banned_words || nil, Beep.banned_urls || nil, Beep.censorWholeWord || nil);
+		return parseContent(content, Beep.banned_words || nil, Beep.banned_urls || nil, Beep.censorWholeWord);
 	},
 	toRegExp: toRegExp,
 	loadList: function (callback) {
@@ -34,21 +34,27 @@ var Beep = {
 				return callback(err);
 			}
 
-			Beep.illegal_words = new RegExp('\\b(?:' + Beep.toRegExp(hash.illegal) + ')\\b', 'ig');
+			Beep.illegal_words = Beep.toRegExp(hash.illegal, true);
 
 			if (hash.id && hash.id.length) {
 				var words = hash.id.split(',').filter(function (word) {
 					return !Beep.illegal_words.test(word);
 				});
-				Beep.banned_words = new RegExp('\\b(?:' + Beep.toRegExp(words) + ')\\b', 'ig');
+				Beep.banned_words = Beep.toRegExp(words, true);
 				Beep.banned_words_raw = hash.id;
 			} else {
-				Beep.banned_words = new RegExp('\\b(?:' + Beep.toRegExp(defaultBanList) + ')\\b', 'ig');
+				Beep.banned_words = Beep.toRegExp(defaultBanList, true);
 				Beep.banned_words_raw = defaultBanList.join(',');
 				winston.info('Default list of Banned Words is enabled. Please go to administration panel to change the list.');
 			}
-			Beep.banned_urls = new RegExp(Beep.toRegExp(hash.urls), 'ig');
+
+			Beep.banned_urls = Beep.toRegExp(hash.urls);
+
 			Beep.censorWholeWord = hash.censorWholeWord === 'on';
+			if (meta.config) {
+				meta.config.beep = meta.config.beep || {};
+				meta.config.beep.censorWholeWord = Beep.censorWholeWord;
+			}
 
 			callback();
 		});
@@ -141,26 +147,6 @@ var Beep = {
 			callback(null, custom_header);
 		}
 	},
-	category: {
-		get: function (data, callback) {
-			var topics = data.category.topics;
-			topics.forEach(function (topic) {
-				topic.title = Beep.parseContent(topic.title);
-				topic.slug = Beep.parseContent(topic.slug);
-			});
-			callback(null, data);
-		},
-		topics: {
-			get: function (data, callback) {
-				data.topics.forEach(function (topic) {
-					topic.title = Beep.parseContent(topic.title);
-					topic.slug = Beep.parseContent(topic.slug);
-					topic.titleRaw = Beep.parseContent(topic.titleRaw);
-				});
-				callback(null, data);
-			}
-		}
-	},
 	post: {
 		getFields: function (data, callback) {
 			if (data.fields.indexOf('content') !== -1) {

lib/parseContent.js

@@ -1,11 +1,17 @@
 'use strict';
 
+var isLatin = /^\w+$/;
+
 function parseContent(content, banned_words, banned_urls, censorWholeWord) {
   if (!content) {
     return content;
   }
 
   function censor(match) {
+    if (!isLatin.test(match)) {
+      return '[censored]';
+    }
+
     var l = match.length;
     var out = match[0];
 
@@ -24,4 +30,4 @@ function parseContent(content, banned_words, banned_urls, censorWholeWord) {
     .replace(banned_urls, '[link removed]');
 }
 
-module.exports = parseContent;
+module.exports = parseContent;

lib/toRegExp.js

@@ -1,15 +1,41 @@
 'use strict';
 
-function toRegExp(arr) {
+var isLatin = /^\w+$/;
+
+function toRegExp(arr, fullWord) {
   if (!Array.isArray(arr)) {
     arr = (arr || '').toString().split(',');
   }
+  arr = arr.filter(Boolean);
+
+  var str;
+  if (fullWord) {
+    var latin = arr.filter(function (word) {
+      return isLatin.test(word);
+    }).map(function (word) {
+      return word.trim().replace(/([-[\]{}()*+?.,\\^$|#\s])/g, '\\$1');
+    }).join('|');
 
-  var str = arr.filter(Boolean).map(function (word) {
-    return word.trim().replace(/([-[\]{}()*+?.,\\^$|#\s])/g, '\\$1');
-  }).join('|');
+    var notLatin = arr.filter(function (word) {
+      return !isLatin.test(word);
+    }).map(function (word) {
+      return word.trim().replace(/([-[\]{}()*+?.,\\^$|#\s])/g, '\\$1');
+    }).join('|');
+
+    if (latin && notLatin) {
+      str = '\\b(?:' + latin + ')\\b|(?:' + notLatin + ')';
+    } else if (latin) {
+      str = '\\b(?:' + latin + ')\\b';
+    } else if (notLatin) {
+      str = notLatin;
+    }
+  } else {
+    str = arr.filter(Boolean).map(function (word) {
+      return word.trim().replace(/([-[\]{}()*+?.,\\^$|#\s])/g, '\\$1');
+    }).join('|');
+  }
 
-  return str || '^(?!x)x';
+  return new RegExp(str || '^(?!x)x', 'ig');
 }
 
-module.exports = toRegExp;
+module.exports = toRegExp;

tests/parseContent.spec.js

@@ -5,8 +5,8 @@ var assert = require('assert');
 var toRegExp = require('../lib/toRegExp');
 var parseContent = require('../lib/parseContent');
 
-var bannedWords = new RegExp('\\b(?:' + toRegExp(['poop', 'shit']) + ')\\b', 'ig');
-var bannedUrls = new RegExp(toRegExp(['http://example.com', 'http://foo.bar']), 'ig');
+var bannedWords = toRegExp(['poop', 'shit'], true);
+var bannedUrls = toRegExp(['http://example.com', 'http://foo.bar']);
 var nil = '^(?!x)x';
 
 assert.strictEqual(parseContent(
@@ -29,3 +29,13 @@ assert.strictEqual(parseContent(
   bannedUrls,
   false
 ), 'My favorite website is [link removed]. I also love [link removed].');
+
+var unicodeBannedWords = toRegExp(['今', '野'], true);
+assert.strictEqual(parseContent(
+  '載点代示早面通今就焼初哲野質',
+  unicodeBannedWords,
+  nil,
+  false
+), '載点代示早面通[censored]就焼初哲[censored]質');
+
+console.log('parseContent passed');

tests/toRegExp.spec.js

@@ -4,6 +4,9 @@ var assert = require('assert');
 
 var toRegExp = require('../lib/toRegExp');
 
-assert.deepStrictEqual(toRegExp(['bad', 'words', 'here']), 'bad|words|here');
-assert.deepStrictEqual(toRegExp('bad,words, here'), 'bad|words|here');
-assert.deepStrictEqual(toRegExp(null), '^(?!x)x');
+assert.equal(toRegExp(['bad', 'words', 'here'], true), '/\\b(?:bad|words|here)\\b/gi');
+assert.equal(toRegExp('bad,words, here'), '/bad|words|here/gi');
+assert.equal(toRegExp('今,野', true), '/今|野/gi');
+assert.equal(toRegExp(null), '/^(?!x)x/gi');
+
+console.log('toRegExp passed');