Fix missing <|eom_id|> token in Llama 3 vocabulary.

mukel · mukel · commit 5f602500beb5 · 2024-07-29T14:43:40.000+02:00
diff --git a/Llama3.java b/Llama3.java
@@ -2045,7 +2045,7 @@ public ChatFormat(Tokenizer tokenizer) {
         this.endHeader = specialTokens.get("<|end_header_id|>");
         this.endOfTurn = specialTokens.get("<|eot_id|>");
         this.endOfText = specialTokens.get("<|end_of_text|>");
-        this.endOfMessage = specialTokens.get("<|eom_id|>");
+        this.endOfMessage = specialTokens.getOrDefault("<|eom_id|>", -1); // only in 3.1
     }
 
     public Tokenizer getTokenizer() {

Original file line number	Diff line number	Diff line change
`@@ -2045,7 +2045,7 @@ public ChatFormat(Tokenizer tokenizer) {`
`2045`	`2045`	`this.endHeader = specialTokens.get("<\|end_header_id\|>");`
`2046`	`2046`	`this.endOfTurn = specialTokens.get("<\|eot_id\|>");`
`2047`	`2047`	`this.endOfText = specialTokens.get("<\|end_of_text\|>");`
`2048`		`- this.endOfMessage = specialTokens.get("<\|eom_id\|>");`
	`2048`	`+ this.endOfMessage = specialTokens.getOrDefault("<\|eom_id\|>", -1); // only in 3.1`
`2049`	`2049`	`}`
`2050`	`2050`
`2051`	`2051`	`public Tokenizer getTokenizer() {`