model improved

maker.py

@@ -1,9 +1,9 @@
 #! /usr/bin/python3
-import os,json
+import os
 src="99eren99/ModernBERT-base-Turkish-uncased-mlm"
 tgt="KoichiYasuoka/modernbert-base-turkish-ud-embeds"
 url="https://github.com/UniversalDependencies/UD_Turkish-"
-for e in ["Kenet","Penn","BOUN","Tourism","IMST","Atis","FrameNet"]:
+for e in ["Kenet","Penn"]:
   u=url+e
   d=os.path.basename(u)
   os.system("test -d "+d+" || git clone --depth=1 "+u)
@@ -41,14 +41,11 @@ class UDEmbedsDataset(object):
   __len__=lambda self:(len(self.seeks)-1)*2
   def __getitem__(self,i):
     self.conllu.seek(self.seeks[int(i/2)])
-    z,c,t,s=i%2,[],[""],False
+    z,c,t=i%2,[],[""]
     while t[0]!="\n":
       t=self.conllu.readline().split("\t") 
       if len(t)==10 and t[0].isdecimal():
-        if s:
-           t[1]=" "+t[1]
         c.append(t)
-        s=t[9].find("SpaceAfter=No")<0
     x=[True if t[6]=="0" or int(t[6])>j or sum([1 if int(c[i][6])==j+1 else 0 for i in range(j+1,len(c))])>0 else False for j,t in enumerate(c)]
     v=self.tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
     if z==0:
@@ -107,8 +104,8 @@ trainDS=UDEmbedsDataset("train.conllu",tkz)
 devDS=UDEmbedsDataset("dev.conllu",tkz)
 testDS=UDEmbedsDataset("test.conllu",tkz)
 lid=trainDS(devDS,testDS)
-cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True)
-mdl=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()})
+mdl=AutoModelForTokenClassification.from_pretrained(src,config=cfg)
 trainDS.embeddings=mdl.get_input_embeddings().weight
 arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
 trn=Trainer(args=arg,data_collator=DefaultDataCollator(),model=mdl,train_dataset=trainDS)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1870ef3ec1d236b23d48f21a0cdecc78dc6cae298e961bf10ee12e1dcbfabd48
-size 592177074
+oid sha256:f4b0b45b340abd523b7ddc8bd994aa6dbc93ac6bebe7c5a0eda8606ebba46500
+size 555858738

tokenizer_config.json

@@ -48,11 +48,11 @@
   "extra_special_tokens": {},
   "mask_token": "[MASK]",
   "max_len": 999999999,
-  "model_max_length": 999999999,
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
+  "model_max_length": 999999999,
   "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",

ud.py

@@ -77,7 +77,7 @@ class UniversalDependenciesPipeline(BellmanFordTokenClassificationPipeline):
         if d.strip()=="":
           off.pop(i)
           w.pop(i)
-    v=self.tokenizer([t["text"] for t in w],add_special_tokens=False)
+    v=self.tokenizer([t["text"].strip() for t in w],add_special_tokens=False)
     x=[not t["entity_group"].endswith(".") for t in w]
     if len(x)<127:
       x=[True]*len(x)