Get vocab_size from token embeddings size

2023-08-26 22:35:55 +03:00
parent 49daf18f2f
commit 2c2b284988
1 changed files with 2 additions and 1 deletions
@@ -323,9 +323,10 @@ def load_meta_model(model_path):
    config.multiple_of = params["multiple_of"]
    config.norm_eps = params["norm_eps"]

-    config.vocab_size = 32000
+    config.vocab_size = state_dict['tok_embeddings.weight'].shape[0]
    config.max_seq_len = 2048

+
    # create a new Transformer object and set weights
    model = Transformer(config)