 
			
		Upload tokenizer
		2084796
		verified
		
				
					
						
						|  | { | 
					
						
						|  | "cls_token": { | 
					
						
						|  | "content": "[CLS]", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | }, | 
					
						
						|  | "mask_token": { | 
					
						
						|  | "content": "[MASK]", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | }, | 
					
						
						|  | "pad_token": { | 
					
						
						|  | "content": "[PAD]", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | }, | 
					
						
						|  | "sep_token": { | 
					
						
						|  | "content": "[SEP]", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | }, | 
					
						
						|  | "unk_token": { | 
					
						
						|  | "content": "[UNK]", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | } | 
					
						
						|  | } | 
					
						
						|  |  |