import gradio as gr import spacy import re import numpy as np import torch from transformers import AutoTokenizer, AutoModelForCausalLM # Define some constants for error messages MAX_TEXT_LENGTH = 2048 MIN_GEN_LENGTH = 1 MAX_GEN_LENGTH = 2048 MIN_AI_PERCENTAGE = 50 # Download the Spacy model for English spacy.cli.download("en_core_web_sm") nlp = spacy.load("en_core_web_sm") # Define a function to detect AI-generated content and calculate the perplexity score, # burstiness score, and average perplexity score for a given text input def detect_ai_content(text, max_gen_length, temperature, model_name, model_size, num_return_sequences, min_ai_percentage): # Clean the text by removing extra spaces, line breaks, and special characters cleaned_text = re.sub(r'\s+', ' ', text).strip() cleaned_text = re.sub(r'[^\w\s]', '', cleaned_text) # If the cleaned text is empty or contains only one sentence, return an error message doc = nlp(cleaned_text) if not re.search('\S', cleaned_text) or len(list(doc.sents)) < 2: return {"error": "Input text must contain at least two sentences."} # Check if the cleaned text is longer than the maximum allowed by the GPT model if len(cleaned_text) > MAX_TEXT_LENGTH: return {"error": f"Input text must be no longer than {MAX_TEXT_LENGTH} characters."} # Check if the minimum threshold for the percentage of AI-generated content is within the allowed range if not (0 <= min_ai_percentage <= 100): return {"error": "Minimum threshold for AI percentage must be between 0 and 100."} # Load the specified GPT model and tokenizer model_name = f"{model_name}-{model_size}" tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForCausalLM.from_pretrained(model_name) # Set the device to run the model on (either "cuda" or "cpu") device = "cuda" if torch.cuda.is_available() else "cpu" model.to(device) # Set the end of sequence token ID for text generation eos_token_id = tokenizer.eos_token_id # Generate multiple sequences using the pre-trained GPT model input_ids = tokenizer.encode(cleaned_text, add_special_tokens=True, return_tensors='pt').to(device) # with gr.progress.Progress("Generating text...", max=num_return_sequences) as progress: output_sequences = [] for i in range(num_return_sequences): output_sequence = model.generate( input_ids=input_ids, max_length=max_gen_length + len(input_ids[0]), temperature=temperature, top_k=50, top_p=0.95, repetition_penalty=1.5, do_sample=True, num_return_sequences=1 )[0] output_sequences.append(output_sequence) # progress.update(i) # Decode the generated sequences using the GPT tokenizer generated_texts = [] perplexities = [] ai_percentages = [] for i, output_sequence in enumerate(output_sequences): generated_text = tokenizer.decode(output_sequence.tolist()[len(input_ids[0]):], skip_special_tokens=True) # Calculate the percentage of AI-generated content in the generated text ai_percentage = round(len(generated_text) / len(cleaned_text) * 100, 2) ai_percentages.append(ai_percentage) # Check if the AI percentage and perplexity score are above their respective thresholds generated_input_ids = tokenizer.encode(generated_text, add_special_tokens=False, return_tensors='pt').to(device) with torch.no_grad(): loss = model(generated_input_ids, labels=generated_input_ids).loss.item() perplexity = np.exp(loss) perplexities.append(perplexity) generated_texts.append(generated_text) # Remove the generated sequences that are identical or highly similar to the cleaned text clean_doc = nlp(cleaned_text) unique_generated_texts = [] for generated_text in generated_texts: gen_doc = nlp(generated_text) similarity = clean_doc.similarity(gen_doc) if similarity < 0.8: is_unique = True for unique_text in unique_generated_texts: unique_doc = nlp(unique_text) unique_similarity = unique_doc.similarity(gen_doc) if unique_similarity >= 0.8: is_unique = False break if is_unique: unique_generated_texts.append(generated_text) # Calculate the burstiness score for the input text, which measures the diversity of vocabulary in the input text doc = nlp(cleaned_text) tokens = [token.text.lower() for token in doc if not token.is_stop and token.is_alpha] unique_tokens = set(tokens) burstiness = len(unique_tokens) / len(tokens) # Check if the minimum threshold for AI percentage is met for any of the generated sequences, # and if the perplexity score is above the minimum threshold valid_generated_texts = [] valid_perplexities = [] valid_ai_percentages = [] for i, generated_text in enumerate(unique_generated_texts): ai_percentage = ai_percentages[i] perplexity = perplexities[i] if ai_percentage >= min_ai_percentage and perplexity > 5: valid_generated_texts.append(generated_text) valid_perplexities.append(perplexity) valid_ai_percentages.append(ai_percentage) # Check if any valid generated sequences were found if not valid_generated_texts: return {"error": "No AI-generated content meeting the minimum threshold was found in the input text."} # Add up to num_return_sequences generated sequences with the highest AI percentages and perplexity scores to the result dictionary max_num_returned_sequences = min(num_return_sequences, len(valid_generated_texts)) combined_scores = [(i, valid_ai_percentages[i] * valid_perplexities[i]) for i in range(len(valid_generated_texts))] sorted_indices = sorted(combined_scores, key=lambda x: x[1], reverse=True) result = { "cleaned_text": cleaned_text, "generated_texts": [], "perplexities": valid_perplexities, "ai_percentages": valid_ai_percentages, "burstiness": burstiness, "avg_perplexity": np.mean(valid_perplexities), } for i in range(max_num_returned_sequences): idx = sorted_indices[i][0] result["generated_texts"].append(valid_generated_texts[idx]) # Format the output for better readability cleaned_text = re.sub(r'\n+', '\n', cleaned_text) generated_texts = [re.sub(r'\n+', '\n', text) for text in result["generated_texts"]] perplexities = [f"{perplexity:.2f}" for perplexity in result["perplexities"]] ai_percentages = [f"{ai_percentage:.2f}%" for ai_percentage in result["ai_percentages"]] avg_perplexity = f"{result['avg_perplexity']:.2f}" burstiness = f"{result['burstiness']:.2f}" return {"cleaned_text": cleaned_text, "generated_texts": generated_texts, "perplexities": perplexities, "ai_percentages": ai_percentages, "burstiness": burstiness, "avg_perplexity": avg_perplexity} # Define the input and output fields for the Gradio interface input_textbox = gr.Textbox(label="Input Text", placeholder="Enter some text here...") max_gen_length_slider = gr.Slider(minimum=MIN_GEN_LENGTH, maximum=MAX_GEN_LENGTH, default=256, label="Max Length for Generated Text") temperature_slider = gr.Slider(minimum=0.1, maximum=2.0, step=0.1, default=1.0, label="Temperature", info="Higher values make the model generate more random text.") model_dropdown = gr.Dropdown(choices=["gpt2", "gpt2-medium", "gpt2-large", "gpt2-xl", "EleutherAI/gpt-neo-125M", "EleutherAI/gpt-neo-1.3B", "EleutherAI/gpt-neo-2.7B"], default="gpt2", label="GPT Model", tooltip="The pre-trained GPT model to use for text generation.") model_size_dropdown = gr.Dropdown(choices=["small", "medium", "large", "xl"], default="medium", label="Model Size", tooltip="The size of the pre-trained GPT model to use for text generation.") num_return_sequences_slider = gr.Slider(minimum=1, maximum=10, default=1, label="Number of Generated Sequences to Return") min_ai_percentage_slider = gr.Slider(minimum=0, maximum=100, default=MIN_AI_PERCENTAGE, label="Minimum Threshold for AI Percentage", info="The minimum percentage of AI-generated content that a generated sequence must have.") # Define custom CSS styling for the Gradio interface css_style = """ .gradio-input-wrapper { margin-bottom: 10px; } .gradio-input-wrapper label { font-size: 1.2rem; font-weight: bold; margin-bottom: 5px; } .gradio-slider { height: 30px; margin-bottom: 15px; } .gradio-slider .input-label-container { display: flex; justify-content: space-between; align-items: center; } .gradio-dropdown { margin-bottom: 20px; } .gradio-textbox textarea { height: 150px; font-size: 1.2rem; } .gradio-output { background-color: #f5f5f5; border-radius: 4px; padding: 15px; margin-top: 30px; margin-bottom: 30px; } .gradio-output label { font-size: 1.2rem; font-weight: bold; margin-bottom: 10px; display: block; } .gradio-output p { margin-bottom: 10px; } .gradio-error { color: red; font-weight: bold; margin-top: 10px; } .gradio-progress { margin-top: 20px; } """ # Define the Gradio interface iface = gr.Interface( detect_ai_content, inputs=[input_textbox, max_gen_length_slider, temperature_slider, model_dropdown, model_size_dropdown, num_return_sequences_slider, min_ai_percentage_slider], outputs=[ gr.Label(label="Input Text"), gr.Label(label="Generated Text"), gr.Label(label="Perplexity Scores"), gr.Label(label="AI Percentages"), gr.Label(label="Burstiness Score"), gr.Label(label="Average Perplexity Score"), ], title="AI Content Detection and Generation", description="This app detects and generates AI-generated content in input text using GPT-2 and GPT-Neo models.", theme="default", layout="vertical", css_style=css_style, ) # Launch the Gradio interface iface.launch()