diff --git a/internlm/model/ops/fusion_ops_import_helper.py b/internlm/model/ops/fusion_ops_import_helper.py
index b28e7df5..f75ff889 100644
--- a/internlm/model/ops/fusion_ops_import_helper.py
+++ b/internlm/model/ops/fusion_ops_import_helper.py
@@ -138,7 +138,8 @@ def try_import_FusedAdamW():
     backend = internlm_accelerator.get_accelerator_backend()
     try:
         if backend is AcceleratorType.GPU:
-            adam_extra_kwargs["fused"] = True
+            if torch.__version__ >= "2.1.0":
+                adam_extra_kwargs["fused"] = True
 
             if gpc.is_rank_for_log():
                 logger.warning(