|
|
"""EZ-Tokenizer - High-performance Python code tokenizer with adaptive resource management. |
|
|
|
|
|
Features: |
|
|
- Efficient tokenization of code and text |
|
|
- Adaptive resource management |
|
|
- Support for large datasets |
|
|
- Custom vocabulary generation |
|
|
""" |
|
|
|
|
|
__version__ = "1.0.0" |
|
|
__author__ = "EZ-Tokenizer Team" |
|
|
__all__ = [ |
|
|
"SystemResources", |
|
|
"log_memory_usage", |
|
|
"manage_ram", |
|
|
"build_tokenizer" |
|
|
] |
|
|
|
|
|
|
|
|
def __getattr__(name): |
|
|
if name == 'SystemResources': |
|
|
from .resources import SystemResources |
|
|
return SystemResources |
|
|
elif name in ('log_memory_usage', 'manage_ram', 'build_tokenizer'): |
|
|
from .adaptive_tokenizer import log_memory_usage, manage_ram, build_tokenizer |
|
|
if name == 'log_memory_usage': |
|
|
return log_memory_usage |
|
|
elif name == 'manage_ram': |
|
|
return manage_ram |
|
|
elif name == 'build_tokenizer': |
|
|
return build_tokenizer |
|
|
|
|
|
raise AttributeError(f"module '{__name__}' has no attribute '{name}'") |
|
|
|
