From 32948ec95268fee21661b7da8194ae3250b563c2 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Wed, 7 May 2025 15:56:42 -0700 Subject: [PATCH] increase rope base --- convert/convert_qwen25vl.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert/convert_qwen25vl.go b/convert/convert_qwen25vl.go index bc21f629b..2a1df0acd 100644 --- a/convert/convert_qwen25vl.go +++ b/convert/convert_qwen25vl.go @@ -48,7 +48,8 @@ func (q *qwen25VLModel) KV(t *Tokenizer) ggml.KV { kv["qwen25vl.vision.spatial_patch_size"] = q.VisionModel.SpatialPatchSize kv["qwen25vl.vision.window_size"] = q.VisionModel.WindowSize kv["qwen25vl.vision.attention.layer_norm_epsilon"] = cmp.Or(q.VisionModel.RMSNormEps, 1e-6) - kv["qwen25vl.vision.rope.freq_base"] = cmp.Or(q.VisionModel.RopeTheta, 1e4) + // RoPE theta increased from 1e4 to 1e5 to compensate for numerical differences between tensor operations; empirically produces better results. + kv["qwen25vl.vision.rope.freq_base"] = cmp.Or(q.VisionModel.RopeTheta, 1e5) kv["qwen25vl.vision.fullatt_block_indexes"] = q.VisionModel.FullAttentionBlocks kv["qwen25vl.vision.temporal_patch_size"] = q.VisionModel.TemporalPatchSize