Upload README.md
Browse files
README.md
CHANGED
|
@@ -56,38 +56,38 @@ Finally, an INT8 quantized version of the Imange Enconder only can be find as op
|
|
| 56 |
|
| 57 |
I suggest to install requirements into a dedicated python-virtualenv or a conda enviroment.
|
| 58 |
|
| 59 |
-
|
| 60 |
pip install -q "torch>=2.1" "torchvision" "Pillow" "tqdm" "datasets>=2.14.6" "gradio>=4.36" "nncf>=2.13.0" --extra-index-url https://download.pytorch.org/whl/cpu
|
| 61 |
|
| 62 |
pip install -q "transformers>=4.45" --extra-index-url https://download.pytorch.org/whl/cpu
|
| 63 |
|
| 64 |
pip install -Uq --pre "openvino>2024.4.0" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
|
| 65 |
-
|
| 66 |
|
| 67 |
### Step 2 Convert the model in OpenVINO Intermediate Representation (IR)
|
| 68 |
|
| 69 |
-
|
| 70 |
from pathlib import Path
|
| 71 |
from ov_mllama_helper import convert_mllama
|
| 72 |
model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"
|
| 73 |
model_dir = Path(model_id.split("/")[-1]) / "OpenVino"
|
| 74 |
convert_mllama(model_id, model_dir)
|
| 75 |
-
|
| 76 |
|
| 77 |
### Step 3 INT4 Compression
|
| 78 |
|
| 79 |
-
|
| 80 |
from ov_mllama_compression import compress
|
| 81 |
from ov_mllama_compression import compression_widgets_helper
|
| 82 |
compression_scenario, compress_args = compression_widgets_helper()
|
| 83 |
compression_scenario
|
| 84 |
compression_kwargs = {key: value.value for key, value in compress_args.items()}
|
| 85 |
language_model_path = compress(model_dir, **compression_kwargs)
|
| 86 |
-
|
| 87 |
|
| 88 |
### Step 4 INT8 Image Enconder Optimization
|
| 89 |
|
| 90 |
-
|
| 91 |
from ov_mllama_compression import vision_encoder_selection_widget
|
| 92 |
vision_encoder_options = vision_encoder_selection_widget(device.value)
|
| 93 |
vision_encoder_options
|
|
@@ -117,7 +117,7 @@ del calibration_dataset
|
|
| 117 |
del calibration_data
|
| 118 |
gc.collect()
|
| 119 |
vision_encoder_path = int8_vision_encoder_path
|
| 120 |
-
|
| 121 |
|
| 122 |
## License
|
| 123 |
|
|
|
|
| 56 |
|
| 57 |
I suggest to install requirements into a dedicated python-virtualenv or a conda enviroment.
|
| 58 |
|
| 59 |
+
```
|
| 60 |
pip install -q "torch>=2.1" "torchvision" "Pillow" "tqdm" "datasets>=2.14.6" "gradio>=4.36" "nncf>=2.13.0" --extra-index-url https://download.pytorch.org/whl/cpu
|
| 61 |
|
| 62 |
pip install -q "transformers>=4.45" --extra-index-url https://download.pytorch.org/whl/cpu
|
| 63 |
|
| 64 |
pip install -Uq --pre "openvino>2024.4.0" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
|
| 65 |
+
```
|
| 66 |
|
| 67 |
### Step 2 Convert the model in OpenVINO Intermediate Representation (IR)
|
| 68 |
|
| 69 |
+
```
|
| 70 |
from pathlib import Path
|
| 71 |
from ov_mllama_helper import convert_mllama
|
| 72 |
model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"
|
| 73 |
model_dir = Path(model_id.split("/")[-1]) / "OpenVino"
|
| 74 |
convert_mllama(model_id, model_dir)
|
| 75 |
+
```
|
| 76 |
|
| 77 |
### Step 3 INT4 Compression
|
| 78 |
|
| 79 |
+
```
|
| 80 |
from ov_mllama_compression import compress
|
| 81 |
from ov_mllama_compression import compression_widgets_helper
|
| 82 |
compression_scenario, compress_args = compression_widgets_helper()
|
| 83 |
compression_scenario
|
| 84 |
compression_kwargs = {key: value.value for key, value in compress_args.items()}
|
| 85 |
language_model_path = compress(model_dir, **compression_kwargs)
|
| 86 |
+
```
|
| 87 |
|
| 88 |
### Step 4 INT8 Image Enconder Optimization
|
| 89 |
|
| 90 |
+
```
|
| 91 |
from ov_mllama_compression import vision_encoder_selection_widget
|
| 92 |
vision_encoder_options = vision_encoder_selection_widget(device.value)
|
| 93 |
vision_encoder_options
|
|
|
|
| 117 |
del calibration_data
|
| 118 |
gc.collect()
|
| 119 |
vision_encoder_path = int8_vision_encoder_path
|
| 120 |
+
```
|
| 121 |
|
| 122 |
## License
|
| 123 |
|