diff --git a/scripts/models/xllm-375B.sh b/scripts/models/xllm-375B.sh
index 049f5070e4..6fddb60644 100644
--- a/scripts/models/xllm-375B.sh
+++ b/scripts/models/xllm-375B.sh
@@ -17,7 +17,7 @@ MODEL_ARGS=(
     --position-embedding-type rope
     --rotary-percent 0.5
     --xllm-partial-rope-layout
-    --rotary-base 500000
+    --rotary-base 10000000
     --swiglu
     --untie-embeddings-and-output-weights
     --vocab-size 250624