Update

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -1,29 +1,37 @@
 {
     "architectures": [
-        "Qwen3ForCausalLM"
+        "Qwen3MoeForCausalLM"
     ],
     "attention_bias": false,
     "attention_dropout": 0.0,
     "bos_token_id": 151643,
+    "decoder_sparse_step": 1,
     "eos_token_id": 151645,
     "head_dim": 128,
     "hidden_act": "silu",
-    "hidden_size": 5120,
+    "hidden_size": 2048,
     "initializer_range": 0.02,
-    "intermediate_size": 25600,
+    "intermediate_size": 6144,
     "max_position_embeddings": 40960,
-    "max_window_layers": 64,
-    "model_type": "qwen3",
-    "num_attention_heads": 64,
-    "num_hidden_layers": 64,
-    "num_key_value_heads": 8,
+    "max_window_layers": 48,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_moe",
+    "moe_intermediate_size": 768,
+    "norm_topk_prob": true,
+    "num_attention_heads": 32,
+    "num_experts": 128,
+    "num_experts_per_tok": 8,
+    "num_hidden_layers": 48,
+    "num_key_value_heads": 4,
+    "output_router_logits": false,
     "rms_norm_eps": 1e-06,
     "rope_scaling": null,
-    "rope_theta": 1000000,
+    "rope_theta": 1000000.0,
+    "router_aux_loss_coef": 0.001,
     "sliding_window": null,
     "tie_word_embeddings": false,
     "torch_dtype": "bfloat16",
-    "transformers_version": "4.51.3",
+    "transformers_version": "4.53.1",
     "use_cache": true,
     "use_sliding_window": false,
     "vocab_size": 151936,
@@ -48,13 +56,65 @@
             }
         },
         "ignore": [
+            "model.layers.0.mlp.gate",
+            "model.layers.1.mlp.gate",
+            "model.layers.10.mlp.gate",
+            "model.layers.11.mlp.gate",
+            "model.layers.12.mlp.gate",
+            "model.layers.13.mlp.gate",
+            "model.layers.14.mlp.gate",
+            "model.layers.15.mlp.gate",
+            "model.layers.16.mlp.gate",
+            "model.layers.17.mlp.gate",
+            "model.layers.18.mlp.gate",
+            "model.layers.19.mlp.gate",
+            "model.layers.2.mlp.gate",
+            "model.layers.20.mlp.gate",
+            "model.layers.21.mlp.gate",
+            "model.layers.22.mlp.gate",
+            "model.layers.23.mlp.gate",
+            "model.layers.24.mlp.gate",
+            "model.layers.25.mlp.gate",
+            "model.layers.26.mlp.gate",
+            "model.layers.27.mlp.gate",
+            "model.layers.28.mlp.gate",
+            "model.layers.29.mlp.gate",
+            "model.layers.3.mlp.gate",
+            "model.layers.30.mlp.gate",
+            "model.layers.31.mlp.gate",
+            "model.layers.32.mlp.gate",
+            "model.layers.33.mlp.gate",
+            "model.layers.34.mlp.gate",
+            "model.layers.35.mlp.gate",
+            "model.layers.36.mlp.gate",
+            "model.layers.37.mlp.gate",
+            "model.layers.38.mlp.gate",
+            "model.layers.39.mlp.gate",
+            "model.layers.4.mlp.gate",
+            "model.layers.40.mlp.gate",
+            "model.layers.41.mlp.gate",
+            "model.layers.42.mlp.gate",
+            "model.layers.43.mlp.gate",
+            "model.layers.44.mlp.gate",
+            "model.layers.45.mlp.gate",
+            "model.layers.46.mlp.gate",
+            "model.layers.47.mlp.gate",
+            "model.layers.5.mlp.gate",
+            "model.layers.6.mlp.gate",
+            "model.layers.7.mlp.gate",
+            "model.layers.8.mlp.gate",
+            "model.layers.9.mlp.gate",
             "lm_head"
         ],
         "quant_algo": "NVFP4",
-        "kv_cache_scheme": "FP8",
+        "kv_cache_scheme": {
+            "dynamic": false,
+            "num_bits": 8,
+            "type": "float"
+        },
         "producer": {
             "name": "modelopt",
-            "version": "0.32.1.dev10+gb3c37252c"
+            "version": "0.34.1.dev85+g7a72957d"
         },
         "quant_method": "modelopt"
     }

generation_config.json

@@ -9,5 +9,5 @@
   "temperature": 0.6,
   "top_k": 20,
   "top_p": 0.95,
-  "transformers_version": "4.51.3"
+  "transformers_version": "4.53.1"
 }

hf_quant_config.json

@@ -1,13 +1,61 @@
 {
     "producer": {
         "name": "modelopt",
-        "version": "0.33.0"
+        "version": "0.34.1.dev85+g7a72957d"
     },
     "quantization": {
         "quant_algo": "NVFP4",
         "kv_cache_quant_algo": "FP8",
         "group_size": 16,
         "exclude_modules": [
+            "model.layers.0.mlp.gate",
+            "model.layers.1.mlp.gate",
+            "model.layers.10.mlp.gate",
+            "model.layers.11.mlp.gate",
+            "model.layers.12.mlp.gate",
+            "model.layers.13.mlp.gate",
+            "model.layers.14.mlp.gate",
+            "model.layers.15.mlp.gate",
+            "model.layers.16.mlp.gate",
+            "model.layers.17.mlp.gate",
+            "model.layers.18.mlp.gate",
+            "model.layers.19.mlp.gate",
+            "model.layers.2.mlp.gate",
+            "model.layers.20.mlp.gate",
+            "model.layers.21.mlp.gate",
+            "model.layers.22.mlp.gate",
+            "model.layers.23.mlp.gate",
+            "model.layers.24.mlp.gate",
+            "model.layers.25.mlp.gate",
+            "model.layers.26.mlp.gate",
+            "model.layers.27.mlp.gate",
+            "model.layers.28.mlp.gate",
+            "model.layers.29.mlp.gate",
+            "model.layers.3.mlp.gate",
+            "model.layers.30.mlp.gate",
+            "model.layers.31.mlp.gate",
+            "model.layers.32.mlp.gate",
+            "model.layers.33.mlp.gate",
+            "model.layers.34.mlp.gate",
+            "model.layers.35.mlp.gate",
+            "model.layers.36.mlp.gate",
+            "model.layers.37.mlp.gate",
+            "model.layers.38.mlp.gate",
+            "model.layers.39.mlp.gate",
+            "model.layers.4.mlp.gate",
+            "model.layers.40.mlp.gate",
+            "model.layers.41.mlp.gate",
+            "model.layers.42.mlp.gate",
+            "model.layers.43.mlp.gate",
+            "model.layers.44.mlp.gate",
+            "model.layers.45.mlp.gate",
+            "model.layers.46.mlp.gate",
+            "model.layers.47.mlp.gate",
+            "model.layers.5.mlp.gate",
+            "model.layers.6.mlp.gate",
+            "model.layers.7.mlp.gate",
+            "model.layers.8.mlp.gate",
+            "model.layers.9.mlp.gate",
             "lm_head"
         ]
     }

model-00001-of-00005.safetensors → model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8a2eae555271d8b16b4ddd52b4bf23dc068e9565616c51e3577fba991f76ced
-size 4974154840
+oid sha256:15bb083c92a763a643972134681a65e0953122df749fb0d236ca905e78e709bd
+size 5002180600

model-00002-of-00005.safetensors → model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66854e851fe054dc7b55bb8c1bbdfae9544d2838931e97bf80398959aa1e383b
-size 4937271816
+oid sha256:ab6edfed51af3b8c6ca57fc0020290f9da253e9eb2170d7ef051517a845c6b82
+size 5002610584

model-00003-of-00005.safetensors → model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:870d285637f49d5dba32fda2ff8fc89cf3bb59e863e2d19749953d50ecc822e0
-size 4937271816
+oid sha256:57417b5d0bd8b092024608fa98ae8af722c05fe2abb3640a25326a9c231ad939
+size 5001923016

model-00004-of-00005.safetensors → model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c1a3a51c42e6507f7fbeeb754fd397cf0958070694ce63d3359aaff5b40f699
-size 4261880984
+oid sha256:06c1a1a674d5cd7ccba0a1cd4c56921b573a8c874e36ef02549913daf3efa501
+size 3089614888

model-00005-of-00005.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fbd6a7104a886641f322332c121ef34cfaa312ed757e3cb019b14c26002a8fb6
-size 1555824768

tokenizer_config.json

@@ -227,7 +227,6 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n        {%- set ns.multi_step_tool = false %}\n        {%- set ns.last_query_index = index %}\n    {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set content = message.content %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in message.content %}\n                {%- set content = message.content.split('</think>')[-1].lstrip('\\n') %}\n                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {%- if loop.last or (not loop.last and reasoning_content) %}\n                {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n            {%- else %}\n                {{- '<|im_start|>' + message.role + '\\n' + content }}\n            {%- endif %}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- endif %}\n{%- endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",