Add some more test cases for tokenization and ascii folding

pelias · Dec 7, 2024 · f038644 · f038644
1 parent 5c90483
commit f038644
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/integration/analyzer_peliasQuery.js b/integration/analyzer_peliasQuery.js
@@ -14,7 +14,9 @@ module.exports.tests.analyze = function(test, common){
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
     assertAnalysis('tokenizer', 'foo-bar baz/42', ['foo','bar','baz','42']);
+    assertAnalysis('tokenizer', 'foo-bar  baz/42', ['foo','bar','baz','42']); // tab instead of space
     assertAnalysis('tokenizer', 'foo---bar baz/42', ['foo','bar','baz','42']);
+    assertAnalysis('tokenizer', 'foo—bar baz/42', ['foobar','baz','42']); // dash is not a hyphen
     assertAnalysis('tokenizer', 'foo-bar baz//42', ['foo','bar','baz','42']);
     assertAnalysis('tokenizer', 'foo bar baz 42', ['foo','bar', 'baz', '42']);
     assertAnalysis('tokenizer', 'foo-bar baz\\42', ['foo', 'bar','baz', '42']);