TheWeeeed commited on
Commit
78ded7b
·
verified ·
1 Parent(s): c065a00

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +14 -0
app.py CHANGED
@@ -348,7 +348,21 @@ def two_stage_qa(question, candidate_paragraphs_str, max_seq_len_mc=512, max_seq
348
  logger.error(f" !!!!!! CRITICAL: Key '{key_to_check}' in feature {i} contains non-integer {elem} (type: {type(elem)}) at index {elem_idx}!")
349
  return f"錯誤: 特徵準備失敗,{key_to_check} 內部有非整數", "N/A", "N/A"
350
  logger.info(f" Feature {i}, key '{key_to_check}' passed inspection. Length: {len(val_list)}")
 
 
 
 
 
 
 
 
 
 
 
 
351
 
 
 
352
  qa_dataloader = DataLoader(
353
  qa_features_dataset, collate_fn=default_data_collator, batch_size=8 # batch_size可以小一些
354
  )
 
348
  logger.error(f" !!!!!! CRITICAL: Key '{key_to_check}' in feature {i} contains non-integer {elem} (type: {type(elem)}) at index {elem_idx}!")
349
  return f"錯誤: 特徵準備失敗,{key_to_check} 內部有非整數", "N/A", "N/A"
350
  logger.info(f" Feature {i}, key '{key_to_check}' passed inspection. Length: {len(val_list)}")
351
+
352
+ model_input_columns = ["input_ids", "attention_mask"]
353
+ if "token_type_ids" in qa_features_dataset.features:
354
+ model_input_columns.append("token_type_ids")
355
+
356
+ try:
357
+ # 從 qa_features_dataset 中提取需要的列來創建新的 Dataset
358
+ # qa_features_dataset 本身就是一個 Dataset 對象
359
+ features_for_dataloader = qa_features_dataset.select_columns(model_input_columns)
360
+ except Exception as e:
361
+ logger.error(f"從 qa_features_dataset 選擇列時出錯: {e}. Features: {qa_features_dataset.features}")
362
+ return f"錯誤: 準備模型輸入時出錯 (列選擇)。 Error: {e}", "N/A", "N/A"
363
 
364
+ logger.info(f"--- In two_stage_qa, about to create DataLoader for question_id: {log_question_id} ---")
365
+
366
  qa_dataloader = DataLoader(
367
  qa_features_dataset, collate_fn=default_data_collator, batch_size=8 # batch_size可以小一些
368
  )