From ee84a460d0bac94678a60a965e9e3da9554b695c Mon Sep 17 00:00:00 2001
From: zhanghonggeng <zhanghonggeng@baidu.com>
Date: Tue, 9 Jun 2026 11:17:36 +0000
Subject: [PATCH] [DO NOT MERGE] GLM-4.5 MoE: fix softmax/bf16 config for
 alignment testing

---
 paddleformers/transformers/glm4_moe/modeling.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/paddleformers/transformers/glm4_moe/modeling.py b/paddleformers/transformers/glm4_moe/modeling.py
index 7e1a7c02a4b..ea28a4c9c94 100644
--- a/paddleformers/transformers/glm4_moe/modeling.py
+++ b/paddleformers/transformers/glm4_moe/modeling.py
@@ -88,6 +88,9 @@ class GLMMoEModelProvider(GPTModelProvider):
     bias_dropout_fusion: bool = True
     moe_expert_fusion: bool = False
 
+    attention_softmax_in_fp32: bool = True
+    bf16: bool = True
+
 
 def eager_attention_forward(
     module: nn.Layer,