Spaces:
Runtime error
Runtime error
File size: 876 Bytes
09366c9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datasets import load_dataset
from transformers import AutoTokenizer
TARGET = 'discourse_effectiveness'
TEXT = "discourse_text"
train_df = pd.read_csv("data/raw_data/train.csv")
test_df = pd.read_csv("data/raw_data/test.csv")
"""
train_df[TARGET].value_counts(ascending=True).plot.barh()
plt.title("Frequency of Classes")
plt.show()
train_df['discourse_type'].value_counts(ascending=True).plot.barh()
plt.title("Frequency of discourse_type")
plt.show()
train_df["Words Per text"] = train_df[TEXT].str.split().apply(len)
train_df.boxplot("Words Per text", by=TARGET, grid=False, showfliers=False,
color="black")
plt.suptitle("")
plt.xlabel("")
plt.show()
"""
model_ckpt = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
tokenizer.model_max_length
pass |