import decorator_utils
import string_utils
-
logger = logging.getLogger(__name__)
def tokenize(self, text: str):
for x in nltk.word_tokenize(text):
- for y in re.split('\W+', x):
+ for y in re.split(r'\W+', x):
yield y
def contains_bad_word(self, text: str) -> bool: