#116 Allow token processing "middleware"

README.md

@@ -15,6 +15,7 @@ Sentiment is a Node.js module that uses the [AFINN-165](http://www2.imm.dtu.dk/p
 
 - [Installation](#installation)
 - [Usage example](#usage-example)
+- [Spell checked example](#spell-checked-example)
 - [Adding new languages](#adding-new-languages)
 - [Adding and overwriting words](#adding-and-overwriting-words)
 - [API Reference](#api-reference)
@@ -36,6 +37,14 @@ var result = sentiment.analyze('Cats are stupid.');
 console.dir(result);    // Score: -2, Comparative: -0.666
 ```
 
+## Spell checked example
+```js
+var Sentiment = require('sentiment');
+var sentiment = new Sentiment();
+var result = sentiment.analyze('Cats are stpid.', { spellCheck: true });
+console.dir(result);    // Score: -2, Comparative: -0.666
+```
+
 ## Adding new languages
 You can add support for a new language by registering it using the `registerLanguage` method:
 
@@ -63,6 +72,12 @@ var frLanguage = {
       }
       return tokenScore;
     }
+  },
+  spellCheck: {
+    apply: function(labels, token) {
+      // Implement a spell checking strategy here
+      return token;
+    }
   }
 };
 sentiment.registerLanguage('fr', frLanguage);

languages/en/distance.js

@@ -0,0 +1,33 @@
+var lev = require('levenshtein');
+var tokenize = require('../../lib/tokenize');
+
+/**
+ * Finds the closest match between a statement and a body of words using
+ * Levenshtein Distance
+ * 
+ * @param  {string} string Input string
+ * @param  {string/array} words List of strings to find closest
+ * @return {string} The closest word in the list
+ */
+module.exports = function(string, words) {
+
+    var shortest = words.toString().length;
+    var bestFit  = '';
+
+    if (typeof words === 'string') {
+        words = tokenize(words);
+    }
+
+    words.forEach(function(word) {
+
+        var distance = lev(string, word);
+
+        if (distance < shortest) {
+            bestFit  = word;
+            shortest = distance;
+        }
+
+    });
+
+    return bestFit;
+};

languages/en/index.js

@@ -1,4 +1,5 @@
 module.exports = {
     labels: require('./labels.json'),
-    scoringStrategy: require('./scoring-strategy')
+    scoringStrategy: require('./scoring-strategy'),
+    spellCheck: require('./spell-check')
 };

languages/en/negation.js

@@ -0,0 +1,37 @@
+/**
+ * Spellchecking library
+ */
+var spelling = require('./spelling');
+
+/**
+ * These words "flip" the sentiment of the following word.
+ */
+var negators = require('./negators.json');
+
+/**
+ * Language labels and scores
+ */
+var labels = require('./labels.json');
+
+/**
+ * Evaluates wether the current token is negated by a previous token
+ * 
+ * @param {array} tokens list of tokens being evaluated
+ * @param {int} pos position of the current word in the tokens list
+ * 
+ * @return {boolean} true if the current pos is being negaed, false otherwise
+ */
+module.exports = function negated(tokens, pos) {
+    while (pos--) {
+        if (negators[tokens[pos]]) {
+            return true;
+        }
+        var word = spelling.getSpellCheckedWord(tokens[pos]);
+        if (negators[word]) {
+            return true;
+        } else if (labels.hasOwnProperty(word)) {
+            return false;
+        }
+    }
+    return false;
+};

languages/en/scoring-strategy.js

@@ -1,10 +1,10 @@
-var negators = require('./negators.json');
+var negated = require('./negation');
 
 module.exports = {
     apply: function(tokens, cursor, tokenScore) {
         if (cursor > 0) {
-            var prevtoken = tokens[cursor - 1];
-            if (negators[prevtoken]) {
+            // Check for negation
+            if (negated(tokens, cursor)) {
                 tokenScore = -tokenScore;
             }
         }

languages/en/spell-check.js

@@ -0,0 +1,5 @@
+var spelling = require('./spelling');
+
+module.exports = function(labels, token) {
+    return spelling.getSpellCheckedAfinnWord(labels, token);
+};

languages/en/spelling.js

@@ -0,0 +1,61 @@
+var read = require('fs').readFileSync;
+var nspell = require('nspell');
+var distance = require('./distance');
+
+function loadDictionary() {
+    var base = require.resolve('dictionary-en-us');
+    var result = {
+        'aff': read(base.replace('.js', '.aff'), 'utf-8'),
+        'dic': read(base.replace('.js', '.dic'), 'utf-8')
+    };
+    return result;
+}
+
+var spell = null;
+
+function getSpellChecker() {
+    if (spell === null) {
+        var dictionaray = loadDictionary();
+        spell = nspell(dictionaray);
+    }
+    return spell;
+}
+
+/**
+ * These two functions atempt to spell check and correct a given word, using
+ * Levenshtein Distance to choose the most appropriate correction.
+ * getSpellCheckedAfinnWord also looks for the word to be present on Afinn
+ */
+module.exports = {
+    getSpellCheckedAfinnWord: function (afinn, word) {
+        var spellChecker = getSpellChecker();
+        if (!afinn.hasOwnProperty(word) && !spellChecker.correct(word)) {
+            var checked = spellChecker.suggest(word);
+            if (checked.length === 0) {
+                return word;
+            } else {
+                var closest = distance(word, checked);
+                if (closest && afinn.hasOwnProperty(closest)) {
+                    return closest;
+                }
+            }
+        }
+        return word;
+    },
+
+    getSpellCheckedWord: function (word) {
+        var spellChecker = getSpellChecker();
+        if (!spellChecker.correct(word)) {
+            var checked = spellChecker.suggest(word);
+            if (checked.length === 0) {
+                return word;
+            } else {
+                var closest = distance(word, checked);
+                if (closest) {
+                    return closest;
+                }
+            }
+        }
+        return word;
+    }
+};

lib/index.js

@@ -63,8 +63,10 @@ Sentiment.prototype.analyze = function (phrase, opts, callback) {
     // Iterate over tokens
     var i = tokens.length;
     while (i--) {
-        var obj = tokens[i];
+        var obj = opts.spellCheck ? languageProcessor
+            .applySpellChecking(languageCode, labels, tokens[i]) : tokens[i];
         if (!labels.hasOwnProperty(obj)) continue;
+
         words.push(obj);
 
         // Apply scoring strategy

lib/language-processor.js

@@ -79,6 +79,14 @@ module.exports = {
         // eslint-disable-next-line max-len
         var scoringStrategy = language.scoringStrategy || defaultScoringStrategy;
         return scoringStrategy.apply(tokens, cursor, tokenScore);
+    },
+
+    /**
+     * Apply language spell checking strategy, if present.
+     */
+    applySpellChecking: function(languageCode, labels, token) {
+        var language = this.getLanguage(languageCode);
+        return language.spellCheck ? language.spellCheck(labels, token) : token;
     }
 };
 

package.json

@@ -36,5 +36,10 @@
   },
   "engines": {
     "node": ">=8.0"
+  },
+  "dependencies": {
+    "dictionary-en-us": "^2.0.0",
+    "levenshtein": "^1.0.5",
+    "nspell": "^2.0.1"
   }
 }

test/integration/sync_corpus.js

@@ -8,7 +8,7 @@ var result = sentiment.analyze(dataset);
 
 test('synchronous corpus', function (t) {
     t.type(result, 'object');
-    t.equal(result.score, -3);
+    t.equal(result.score, -19);
     t.equal(result.tokens.length, 1416);
     t.equal(result.words.length, 73);
     t.end();

test/integration/sync_negation_backwards.js

@@ -0,0 +1,15 @@
+var test = require('tap').test;
+var Sentiment = require('../../lib/index');
+var sentiment = new Sentiment();
+
+var input = 'this is not very bad';
+var result = sentiment.analyze(input);
+
+test('synchronous negation', function (t) {
+    t.type(result, 'object');
+    t.equal(result.score, 3);
+    t.equal(result.comparative, 0.6);
+    t.equal(result.tokens.length, 5);
+    t.equal(result.words.length, 1);
+    t.end();
+});

test/integration/sync_spell_check.js

@@ -0,0 +1,26 @@
+var test = require('tap').test;
+var Sentiment = require('../../lib/index');
+var sentiment = new Sentiment();
+
+var input = 'I hatee you';
+var result = sentiment.analyze(input, { spellCheck: true });
+
+test('synchronous spell checking active', function (t) {
+    t.type(result, 'object');
+    t.equal(result.score, -3);
+    t.equal(result.comparative, -1);
+    t.equal(result.tokens.length, 3);
+    t.equal(result.words.length, 1);
+    t.end();
+});
+
+result = sentiment.analyze(input);
+
+test('synchronous spell checking inactive', function (t) {
+    t.type(result, 'object');
+    t.equal(result.score, 0);
+    t.equal(result.comparative, 0);
+    t.equal(result.tokens.length, 3);
+    t.equal(result.words.length, 0);
+    t.end();
+});