Hugging Face's logo

tiny-random
/
kormo

Text Generation
Transformers
Safetensors
kormo
conversational
custom_code
Model card Files
xet
Community
kormo / tokenizer_config.json
yujiepan's picture
yujiepan
Upload folder using huggingface_hub
a52c335 verified
raw
history blame contribute delete
8.33 kB
 {
"added_tokens_decoder": {
"125000": {
"content": "\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125001": {
"content": "\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125002": {
"content": "\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125003": {
"content": "\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125004": {
"content": "\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125005": {
"content": "\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125006": {
"content": "\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125007": {
"content": "\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125008": {
"content": "\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125009": {
"content": "\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125010": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125011": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125012": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125013": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125014": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125015": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125016": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125017": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125018": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125019": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125020": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125021": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125022": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125023": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125024": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125025": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125026": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125027": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125028": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125029": {
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125030": {
"content": "<|BOS|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"125031": {
"content": "<|EOS|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"125032": {
"content": "<|PAD|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"125033": {
"content": "<think>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"125034": {
"content": "</think>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"125035": {
"content": "<tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125036": {
"content": "</tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125037": {
"content": "<tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125038": {
"content": "</tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"125039": {
"content": "<|BOT|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"125040": {
"content": "<|EOT|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|BOS|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|EOT|>",
"extra_special_tokens": {},
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|PAD|>",
"tokenizer_class": "PreTrainedTokenizerFast"
}