strip lines
This commit is contained in:
parent
7675c0a910
commit
0ac6024b63
2 changed files with 1 additions and 1 deletions
Binary file not shown.
|
|
@ -24,7 +24,7 @@ def tokenize(lines: list[str]) -> list[str]:
|
||||||
for line in lines:
|
for line in lines:
|
||||||
word_type = CharType.NONE
|
word_type = CharType.NONE
|
||||||
current_word = []
|
current_word = []
|
||||||
for i, char in enumerate(line.lower()):
|
for i, char in enumerate(line.strip().lower()):
|
||||||
if len(current_word) == 0:
|
if len(current_word) == 0:
|
||||||
ct = char_type(char)
|
ct = char_type(char)
|
||||||
if ct == CharType.SPACE:
|
if ct == CharType.SPACE:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue