Bump Megatron-LM to v0.18.2-openeurollm and migrate configs Moves the submodule from 055f7defc (v0.15-openeurollm) to 5f3865f7c (v0.18.2-openeurollm) and compatible.

.gitignore

@@ -82,38 +82,6 @@ target/
 profile_default/
 ipython_config.py
 
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-# Pipfile.lock
-
-# UV
-#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-# uv.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-# poetry.lock
-# poetry.toml
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
-#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
-# pdm.lock
-# pdm.toml
 .pdm-python
 .pdm-build/
 
@@ -182,25 +150,14 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-# PyCharm
-#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#   and can be added to the global gitignore or merged into this file.  For a more nuclear
-#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
-# .idea/
+# Idea
+.idea/
 
 # Abstra
-#   Abstra is an AI-powered process automation framework.
-#   Ignore directories containing user credentials, local state, and settings.
-#   Learn more at https://abstra.io/docs
 .abstra/
 
-# Visual Studio Code
-#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
-#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
-#   and can be added to the global gitignore or merged into this file. However, if you prefer,
-#   you could uncomment the following to ignore the entire vscode folder
-# .vscode/
+# VSCode
+.vscode/
 
 # Ruff stuff:
 .ruff_cache/
@@ -237,8 +194,6 @@ dump/
 
 # SLURM artefacts left in the working tree by ad-hoc sbatch runs
 slurm-*.out
-*_[0-9]*.out
-*_[0-9]*.err
 
 # Weights & Biases local run cache
 wandb/

config/backend/megadlm_torchrun.yaml

@@ -0,0 +1,39 @@
+defaults:
+  - megatron: base
+  - _self_
+
+class_name: MegatronBackend
+
+python_cmd: "python"  # not used anyways
+full_schema_validation: false
+launcher_script: "./submodules/MegaDLMs/pretrain_difflm.py"
+dist_cmd: "torchrun ${oc.join:'',${oc.mapcondtmpl:'\\-\\-.*','\\\n  %',' %',\
+ ${oc.cliargs:${.torchrun_args}}}}\\\n   ${.launcher_script}"
+torchrun_args:
+  nproc_per_node: ${slurm.sbatch.gpus_per_node}
+  nnodes: ${slurm.sbatch.nodes}
+  max_restarts: 0
+  tee: 3
+  rdzv_endpoint: "$MASTER_ADDR:$MASTER_PORT"
+  rdzv_backend: "static"
+  node_rank: "$SLURM_NODEID"
+  master_addr: "$MASTER_ADDR"
+  master_port: "$MASTER_PORT"
+env:
+  PYTHONPATH: ".:submodules/MegaDLMs"
+  RUN_DIR: ${.PROJECT_DIR}/submodules/MegaDLMs
+  PROJECT_DIR: ${oc.env:PROJECT_DIR,.}
+  HF_HOME: ${oc.env:HF_HOME,"/scratch-shared/${oc.env:USER}/hf"}
+
+# MegaDLMs-only CLI flags, appended verbatim to full_cmd. They cannot go through
+# the megatron arg mapping: build_cmdline_args drops any key that is not in the
+# stock Megatron argparser, and these are added by MegaDLMs'
+# custom_args/difflm.py extra_args_provider. `aux` is the declared free-form
+# slot on MegatronBackendConfig for exactly this kind of untyped variable --
+# putting the key at the top level only works by grace of NonStrictDataclass.
+# Override per-experiment via  backend: {aux: {difflm_args: "..."}}
+aux:
+  difflm_args: ""
+
+# map megatron command to
+full_cmd: "${.dist_cmd} ${oc.join:'',${oc.mapcondtmpl:'\\-\\-.*','\\\n   %',' %',${oc.argsmegatron:${backend.megatron}}}} ${.aux.difflm_args}"

config/backend/megatron/baby/models/qwen3_dense_9b.yaml

@@ -19,7 +19,6 @@ apply_rope_fusion: true
 gradient_accumulation_fusion: true
 bias_dropout_fusion: true
 masked_softmax_fusion: true
-async_tensor_model_parallel_allreduce: true
 accumulate_allreduce_grads_in_fp32: true
 
 # Attention
@@ -38,7 +37,7 @@ untie_embeddings_and_output_weights: true
 
 # Normalization
 normalization: "RMSNorm"
-norm_epsilon: 1.e-5 # rmsnorm epsilon
+layernorm_epsilon: 1.e-5 # rmsnorm epsilon
 qk_layernorm: True # enable qk-norm
 bf16: True