remove unnecessary cache in simple tokenizer
This commit is contained in:
parent
a1d071733d
commit
77c7ef03fd
|
|
@ -7,7 +7,6 @@ import ftfy
|
|||
import regex as re
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def default_bpe():
|
||||
return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue