retokenize

scripts.fix_tokenization.retokenize(i, start, end, terminals, preterminals, preterminal_parents, passage_id, tokenizer, state, cw, words)[source]