diff --git a/src/encoder.py b/src/encoder.py index 5068cc6..03e0ce2 100644 --- a/src/encoder.py +++ b/src/encoder.py @@ -108,7 +108,7 @@ class Encoder: def get_encoder(model_name): with open(os.path.join('models', model_name, 'encoder.json'), 'r') as f: encoder = json.load(f) - with open(os.path.join('models', model_name, 'vocab.bpe'), 'r') as f: + with open(os.path.join('models', model_name, 'vocab.bpe'), 'r', encoding="utf-8") as f: bpe_data = f.read() bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]] return Encoder(