forked from explosion/spaCy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_whitespace.py
45 lines (31 loc) · 1.26 KB
/
test_whitespace.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import pytest
@pytest.mark.parametrize("text", ["lorem ipsum"])
def test_tokenizer_splits_single_space(tokenizer, text):
tokens = tokenizer(text)
assert len(tokens) == 2
@pytest.mark.parametrize("text", ["lorem ipsum"])
def test_tokenizer_splits_double_space(tokenizer, text):
tokens = tokenizer(text)
assert len(tokens) == 3
assert tokens[1].text == " "
@pytest.mark.parametrize("text", ["lorem ipsum "])
def test_tokenizer_handles_double_trailing_ws(tokenizer, text):
tokens = tokenizer(text)
assert repr(tokens.text_with_ws) == repr(text)
@pytest.mark.parametrize("text", ["lorem\nipsum"])
def test_tokenizer_splits_newline(tokenizer, text):
tokens = tokenizer(text)
assert len(tokens) == 3
assert tokens[1].text == "\n"
@pytest.mark.parametrize("text", ["lorem \nipsum"])
def test_tokenizer_splits_newline_space(tokenizer, text):
tokens = tokenizer(text)
assert len(tokens) == 3
@pytest.mark.parametrize("text", ["lorem \nipsum"])
def test_tokenizer_splits_newline_double_space(tokenizer, text):
tokens = tokenizer(text)
assert len(tokens) == 3
@pytest.mark.parametrize("text", ["lorem \n ipsum"])
def test_tokenizer_splits_newline_space_wrap(tokenizer, text):
tokens = tokenizer(text)
assert len(tokens) == 3