File size: 1,368 Bytes
4265aea |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
"""Setup script for NexForge Tokenizer Builder."""
from setuptools import setup, find_packages
with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read()
setup(
name="nexforgetokenizer",
version="0.1.0",
author="NexForge Team",
description="High-performance tool for creating Python code tokenizers with adaptive resource management",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/nexforge/nexforgetokenizer",
package_dir={"": "src"},
packages=find_packages(where="src"),
python_requires=">=3.8",
install_requires=[
"torch>=1.9.0",
"tokenizers>=0.12.0",
"tqdm>=4.62.0",
"psutil>=5.9.0",
"numpy>=1.20.0", # Optional but recommended for improved performance
],
extras_require={
"dev": [
"pytest>=6.0",
"black>=21.7b0",
"isort>=5.0.0",
"mypy>=0.910",
"pylint>=2.11.0",
],
},
classifiers=[
"Programming Language :: Python :: 3",
"License :: Other/Proprietary License",
"Operating System :: OS Independent",
"Intended Audience :: Developers",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Text Processing :: Linguistic",
],
)
|