File size: 7,931 Bytes
4265aea |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 |
import os
import sys
from pathlib import Path
from tokenizers import Tokenizer
from typing import Optional, Tuple, List, Dict, Any
import json
def get_project_root() -> Path:
"""Get the project root directory."""
# Use the current working directory as the project root
return Path.cwd()
def setup_paths() -> Tuple[Path, Path, Path]:
"""Set up and validate required paths.
Returns:
Tuple containing (tokenizer_path, data_dir, output_dir)
"""
root = get_project_root()
# Define paths - look in root directory (one level up from Test_tokenizer)
tokenizer_path = root.parent / 'output' / 'tokenizer.json'
data_dir = root.parent / 'Dataset' # Look in root directory
output_dir = root.parent / 'test_result' # Output to root directory
# Create output directory if it doesn't exist
output_dir.mkdir(parents=True, exist_ok=True)
output_dir.mkdir(parents=True, exist_ok=True)
# Validate paths
if not tokenizer_path.exists():
print(f"Error: Tokenizer not found at {tokenizer_path}")
sys.exit(1)
if not data_dir.exists():
print(f"Error: Data directory not found at {data_dir}")
sys.exit(1)
return tokenizer_path, data_dir, output_dir
def get_first_chunk_file(data_dir: Path) -> Optional[Path]:
"""Get the first chunk file from the data directory."""
# Look for .txt files in the data directory
chunk_files = sorted(list(data_dir.glob('*.txt')))
if not chunk_files:
print(f"Error: No .txt files found in {data_dir}")
return None
return chunk_files[0] # Return the first chunk file
def test_tokenizer_on_chunk(tokenizer: Tokenizer, chunk_path: Path, max_lines: int = 1000) -> Dict[str, Any]:
"""Test the tokenizer on the first max_lines of a chunk file."""
results = {
'total_lines': 0,
'lines_processed': 0,
'total_tokens': 0,
'perfect_matches': 0,
'total_chars': 0,
'total_diff_chars': 0,
'lines': []
}
try:
with open(chunk_path, 'r', encoding='utf-8') as f:
for i, line in enumerate(f):
if i >= max_lines:
break
line = line.strip()
if not line: # Skip empty lines
continue
# Tokenize and decode
encoding = tokenizer.encode(line)
decoded = tokenizer.decode(encoding.ids)
# Calculate differences
diff_chars = sum(1 for a, b in zip(line, decoded) if a != b)
diff_chars += abs(len(line) - len(decoded))
is_perfect = diff_chars == 0
# Update results
results['total_lines'] += 1
results['lines_processed'] += 1
results['total_tokens'] += len(encoding.tokens)
results['total_chars'] += len(line)
results['total_diff_chars'] += diff_chars
results['perfect_matches'] += 1 if is_perfect else 0
# Store detailed results for the first few lines
if i < 5: # First 5 lines
results['lines'].append({
'original': line[:200] + ('...' if len(line) > 200 else ''),
'decoded': decoded[:200] + ('...' if len(decoded) > 200 else ''),
'tokens': encoding.tokens[:10], # First 10 tokens
'is_perfect': is_perfect,
'diff_chars': diff_chars,
'similarity': 1 - (diff_chars / max(len(line), 1))
})
# Print progress
if (i + 1) % 100 == 0:
print(f"Processed {i+1} lines...")
except Exception as e:
print(f"Error processing file: {e}")
return results
return results
def print_summary(results: Dict[str, Any], output_path: Path) -> None:
"""Print and save test summary in TXT format with script name in the filename."""
if not results['lines_processed']:
print("No lines were processed.")
return
# Calculate statistics
avg_tokens_per_line = results['total_tokens'] / results['lines_processed']
total_chars = results['total_chars']
total_diff_chars = results['total_diff_chars']
accuracy = 1 - (total_diff_chars / total_chars) if total_chars > 0 else 0
diff_percentage = (total_diff_chars / total_chars * 100) if total_chars > 0 else 0
# Get script name without extension
script_name = Path(__file__).stem
# Prepare summary text
summary = [
"="*80,
"TOKENIZER TEST SUMMARY",
"="*80,
f"Test Script: {script_name}.py",
f"Timestamp: {results.get('timestamp', 'N/A')}",
f"Tokenizer: {results.get('tokenizer_path', 'N/A')}",
f"Chunk file: {results.get('chunk_file', 'N/A')}",
"-"*80,
f"Lines processed: {results['lines_processed']}",
f"Perfect matches: {results['perfect_matches']} ({results['perfect_matches']/results['lines_processed']*100:.1f}%)",
f"Average tokens/line: {avg_tokens_per_line:.2f}",
f"Total characters: {total_chars:,}",
f"Total tokens: {results['total_tokens']:,}",
f"Character accuracy: {accuracy*100:.2f}%",
f"Character diff: {total_diff_chars:,} chars ({diff_percentage:.4f}%)",
f"Chars per token: {total_chars/results['total_tokens']:.2f} (lower is better)",
"\nSAMPLE LINES:",
"-"*40
]
# Add sample lines
for i, line in enumerate(results.get('lines', [])[:3]):
summary.extend([
f"\nSAMPLE {i+1}:",
f"Original: {line.get('original', '')}",
f"Decoded: {line.get('decoded', '')}",
f"Tokens: {', '.join(line.get('tokens', [])[:8])}{'...' if len(line.get('tokens', [])) > 8 else ''}",
f"Match: {'✓ Perfect' if line.get('is_perfect') else '✗ Different'}",
"-"*40
])
# Print to console
print("\n".join(summary))
# Save as TXT with script name in filename
timestamp = results.get('timestamp', '')
output_file = output_path / f'{script_name}_result_{timestamp}.txt'
with open(output_file, 'w', encoding='utf-8') as f:
f.write("\n".join(summary))
print(f"\nResults saved to: {output_file}")
def main():
# Set up paths
tokenizer_path, data_dir, output_dir = setup_paths()
# Get the first chunk file
chunk_path = get_first_chunk_file(data_dir)
if not chunk_path:
print(f"No files found in {data_dir}. Please ensure the Dataset directory contains text files.")
return
print(f"Found data directory: {data_dir}")
print(f"Output directory: {output_dir}")
print(f"Testing tokenizer on first 1000 lines of: {chunk_path.name}")
# Load the tokenizer
print(f"Loading tokenizer from: {tokenizer_path}")
tokenizer = Tokenizer.from_file(str(tokenizer_path))
# Get vocabulary info
vocab = tokenizer.get_vocab()
print(f"Vocabulary size: {len(vocab):,} tokens")
# Test tokenizer on the chunk
print("\nTesting tokenizer on chunk...")
results = test_tokenizer_on_chunk(tokenizer, chunk_path, max_lines=1000)
# Add timestamp and tokenizer info to results
import time
results['timestamp'] = time.strftime("%Y%m%d_%H%M%S")
results['tokenizer_path'] = str(tokenizer_path)
results['chunk_file'] = str(chunk_path.name)
# Print and save summary
print_summary(results, output_dir)
print("\nTest complete!")
if __name__ == "__main__":
main()
|