|
|
--- |
|
|
library_name: transformers |
|
|
pipeline_tag: image-text-to-text |
|
|
inference: true |
|
|
widget: |
|
|
- text: Hello! |
|
|
example_title: Hello world |
|
|
group: Python |
|
|
base_model: |
|
|
- lmms-lab/LLaVA-OneVision-1.5-8B-Instruct |
|
|
--- |
|
|
|
|
|
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [lmms-lab/LLaVA-OneVision-1.5-8B-Instruct](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-8B-Instruct). |
|
|
|
|
|
### Example usage: |
|
|
|
|
|
```python |
|
|
from transformers import AutoTokenizer, AutoProcessor, AutoModelForCausalLM |
|
|
from qwen_vl_utils import process_vision_info |
|
|
import torch |
|
|
|
|
|
# fix flash_attn_varlen_func, see https://github.com/EvolvingLMMs-Lab/LLaVA-OneVision-1.5/pull/33/files |
|
|
from flash_attn.flash_attn_interface import flash_attn_varlen_func |
|
|
import transformers |
|
|
transformers.modeling_flash_attention_utils.flash_attn_varlen_func = flash_attn_varlen_func |
|
|
|
|
|
model_id = "tiny-random/llava-onevision-1.5" |
|
|
model = AutoModelForCausalLM.from_pretrained( |
|
|
model_id, dtype=torch.bfloat16, device_map="cuda", trust_remote_code=True |
|
|
) |
|
|
processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True) |
|
|
|
|
|
messages = [ |
|
|
{ |
|
|
"role": "user", |
|
|
"content": [ |
|
|
{ |
|
|
"type": "image", |
|
|
"image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg", |
|
|
}, |
|
|
{"type": "text", "text": "Describe this image."}, |
|
|
], |
|
|
} |
|
|
] |
|
|
|
|
|
# Preparation for inference |
|
|
text = processor.apply_chat_template( |
|
|
messages, tokenize=False, add_generation_prompt=True |
|
|
) |
|
|
image_inputs, video_inputs = process_vision_info(messages) |
|
|
inputs = processor( |
|
|
text=[text], |
|
|
images=image_inputs, |
|
|
videos=video_inputs, |
|
|
padding=True, |
|
|
return_tensors="pt", |
|
|
) |
|
|
inputs = inputs.to("cuda") |
|
|
|
|
|
# Inference: Generation of the output |
|
|
generated_ids = model.generate(**inputs, max_new_tokens=32) |
|
|
generated_ids_trimmed = [ |
|
|
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) |
|
|
] |
|
|
output_text = processor.batch_decode( |
|
|
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False |
|
|
) |
|
|
print(output_text) |
|
|
``` |
|
|
|
|
|
### Codes to create this repo: |
|
|
|
|
|
```python |
|
|
import json |
|
|
from pathlib import Path |
|
|
|
|
|
import accelerate |
|
|
import torch |
|
|
from huggingface_hub import file_exists, hf_hub_download |
|
|
from transformers import ( |
|
|
AutoConfig, |
|
|
AutoModelForCausalLM, |
|
|
AutoProcessor, |
|
|
GenerationConfig, |
|
|
AutoModelForImageTextToText, |
|
|
set_seed, |
|
|
) |
|
|
# fix flash_attn_varlen_func, see https://github.com/EvolvingLMMs-Lab/LLaVA-OneVision-1.5/pull/33/files |
|
|
from flash_attn.flash_attn_interface import flash_attn_varlen_func |
|
|
import transformers |
|
|
transformers.modeling_flash_attention_utils.flash_attn_varlen_func = flash_attn_varlen_func |
|
|
|
|
|
source_model_id = "lmms-lab/LLaVA-OneVision-1.5-8B-Instruct" |
|
|
save_folder = "/tmp/tiny-random/llava-onevision-1.5" |
|
|
|
|
|
processor = AutoProcessor.from_pretrained(source_model_id, trust_remote_code=True) |
|
|
processor.save_pretrained(save_folder) |
|
|
|
|
|
with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f: |
|
|
config_json = json.load(f) |
|
|
for k, v in config_json['auto_map'].items(): |
|
|
config_json['auto_map'][k] = f'{source_model_id}--{v}' |
|
|
|
|
|
config_json['text_config'].update({ |
|
|
'head_dim': 32, |
|
|
'hidden_size': 8, |
|
|
'intermediate_size': 64, |
|
|
'num_hidden_layers': 2, |
|
|
'num_attention_heads': 8, |
|
|
'num_key_value_heads': 4, |
|
|
'layer_types': ['full_attention'] * 2, |
|
|
'max_window_layers': 2, |
|
|
}) |
|
|
config_json['vision_config'].update( |
|
|
{ |
|
|
'depth': 2, |
|
|
'intermediate_size': 256, |
|
|
'embed_dim': 32 * 4, |
|
|
'hidden_size': 32 * 4, |
|
|
'text_hidden_size': 8, |
|
|
'num_heads': 4, |
|
|
'num_hidden_layers': 2, |
|
|
} |
|
|
) |
|
|
with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f: |
|
|
json.dump(config_json, f, indent=2) |
|
|
|
|
|
config = AutoConfig.from_pretrained( |
|
|
save_folder, |
|
|
trust_remote_code=True, |
|
|
) |
|
|
print(config) |
|
|
torch.set_default_dtype(torch.bfloat16) |
|
|
model = AutoModelForCausalLM.from_config(config, trust_remote_code=True).to(torch.bfloat16) |
|
|
torch.set_default_dtype(torch.float32) |
|
|
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'): |
|
|
model.generation_config = GenerationConfig.from_pretrained( |
|
|
source_model_id, trust_remote_code=True, |
|
|
) |
|
|
model.generation_config.do_sample = True |
|
|
print(model.generation_config) |
|
|
model = model.cpu() |
|
|
with torch.no_grad(): |
|
|
for name, p in sorted(model.named_parameters()): |
|
|
torch.nn.init.normal_(p, 0, 0.1) |
|
|
print(name, p.shape) |
|
|
model.save_pretrained(save_folder) |
|
|
|
|
|
def modify_automap(path, source_model_id): |
|
|
import json |
|
|
with open(path, 'r', encoding='utf-8') as f: |
|
|
content = json.load(f) |
|
|
automap = {} |
|
|
if content.get('auto_map', None) is not None: |
|
|
for key, value in content.get('auto_map').items(): |
|
|
if isinstance(value, str): |
|
|
value = source_model_id + '--' + value.split('--')[-1] |
|
|
else: |
|
|
value = [(source_model_id + '--' + v.split('--')[-1]) if '.' in str(v) else v for v in value] |
|
|
automap[key] = value |
|
|
with open(path, 'w', encoding='utf-8') as f: |
|
|
json.dump({**content, 'auto_map': automap}, f, indent=2) |
|
|
|
|
|
modify_automap(f"{save_folder}/config.json", source_model_id) |
|
|
# modify_automap(f'{save_folder}/processor_config.json', source_model_id) |
|
|
# modify_automap(f'{save_folder}/preprocessor_config.json', source_model_id) |
|
|
# modify_automap(f'{save_folder}/tokenizer_config.json', source_model_id) |
|
|
for python_file in Path(save_folder).glob('*.py'): |
|
|
python_file.unlink() |
|
|
``` |
|
|
|
|
|
### Printing the model: |
|
|
|
|
|
```text |
|
|
LLaVAOneVision1_5_ForConditionalGeneration( |
|
|
(model): LLaVAOneVision1_5_Model( |
|
|
(visual): RiceTransformerPretrainedModel( |
|
|
(patch_embed): RicePatchEmbed( |
|
|
(proj): Conv2d(3, 128, kernel_size=(14, 14), stride=(14, 14), bias=False) |
|
|
) |
|
|
(rotary_pos_emb): RiceRotaryEmbedding() |
|
|
(pre_layernorm): LayerNorm((128,), eps=1e-05, elementwise_affine=True) |
|
|
(blocks): ModuleList( |
|
|
(0-1): 2 x RiceBlock( |
|
|
(norm1): LayerNorm((128,), eps=1e-05, elementwise_affine=True) |
|
|
(norm2): LayerNorm((128,), eps=1e-05, elementwise_affine=True) |
|
|
(attn): RiceSdpaAttention( |
|
|
(qkv): Linear(in_features=128, out_features=384, bias=True) |
|
|
(proj): Linear(in_features=128, out_features=128, bias=True) |
|
|
) |
|
|
(mlp): RiceMlp( |
|
|
(fc1): Linear(in_features=128, out_features=256, bias=True) |
|
|
(act): GELUActivation() |
|
|
(fc2): Linear(in_features=256, out_features=128, bias=True) |
|
|
) |
|
|
) |
|
|
) |
|
|
(merger): RicePatchMerger( |
|
|
(ln_q): LayerNorm((128,), eps=1e-05, elementwise_affine=True) |
|
|
(mlp): Sequential( |
|
|
(0): Linear(in_features=512, out_features=512, bias=True) |
|
|
(1): GELU(approximate='none') |
|
|
(2): Linear(in_features=512, out_features=8, bias=True) |
|
|
) |
|
|
) |
|
|
) |
|
|
(language_model): LLaVAOneVision1_5_TextModel( |
|
|
(embed_tokens): Embedding(151936, 8) |
|
|
(layers): ModuleList( |
|
|
(0-1): 2 x LLaVAOneVision1_5_DecoderLayer( |
|
|
(self_attn): LLaVAOneVision1_5_SdpaAttention( |
|
|
(q_proj): Linear(in_features=8, out_features=256, bias=False) |
|
|
(k_proj): Linear(in_features=8, out_features=128, bias=False) |
|
|
(v_proj): Linear(in_features=8, out_features=128, bias=False) |
|
|
(o_proj): Linear(in_features=256, out_features=8, bias=False) |
|
|
(q_norm): LLaVAOneVision1_5_RMSNorm((32,), eps=1e-06) |
|
|
(k_norm): LLaVAOneVision1_5_RMSNorm((32,), eps=1e-06) |
|
|
) |
|
|
(mlp): LLaVAOneVision1_5_MLP( |
|
|
(gate_proj): Linear(in_features=8, out_features=64, bias=False) |
|
|
(up_proj): Linear(in_features=8, out_features=64, bias=False) |
|
|
(down_proj): Linear(in_features=64, out_features=8, bias=False) |
|
|
(act_fn): SiLU() |
|
|
) |
|
|
(input_layernorm): LLaVAOneVision1_5_RMSNorm((8,), eps=1e-06) |
|
|
(post_attention_layernorm): LLaVAOneVision1_5_RMSNorm((8,), eps=1e-06) |
|
|
) |
|
|
) |
|
|
(norm): LLaVAOneVision1_5_RMSNorm((8,), eps=1e-06) |
|
|
(rotary_emb): LLaVAOneVision1_5_RotaryEmbedding() |
|
|
) |
|
|
) |
|
|
(lm_head): Linear(in_features=8, out_features=151936, bias=False) |
|
|
) |
|
|
``` |
