Spaces:
Configuration error
Configuration error
| import torch | |
| from PIL import Image | |
| from transformers import AutoModel, AutoTokenizer | |
| from .base import BaseModel | |
| from ..smp import * | |
| from ..utils import DATASET_TYPE | |
| class MiniCPM_V(BaseModel): | |
| INSTALL_REQ = False | |
| INTERLEAVE = False | |
| def __init__(self, model_path='openbmb/MiniCPM-V', **kwargs): | |
| assert model_path is not None | |
| self.model_path = model_path | |
| print(f'load from {self.model_path}') | |
| self.model = AutoModel.from_pretrained(self.model_path, trust_remote_code=True) | |
| self.model = self.model.to(dtype=torch.bfloat16) | |
| self.model.eval().cuda() | |
| self.kwargs = kwargs | |
| self.tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True) | |
| torch.cuda.empty_cache() | |
| self.num_beams = 1 if self.model_path == 'openbmb/MiniCPM-V' else 3 | |
| def use_custom_prompt(self, dataset): | |
| assert dataset is not None | |
| if listinstr(['MMMU'], dataset): | |
| return True | |
| return False | |
| def build_prompt(self, line, dataset=None): | |
| assert dataset is None or isinstance(dataset, str) | |
| assert self.use_custom_prompt(dataset) | |
| tgt_path = self.dump_image(line, dataset) | |
| question = line['question'] | |
| options = { | |
| cand: line[cand] | |
| for cand in string.ascii_uppercase | |
| if cand in line and not pd.isna(line[cand]) | |
| } | |
| options_prompt = 'Options:\n' | |
| for key, item in options.items(): | |
| options_prompt += f'{key}. {item}\n' | |
| hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None | |
| prompt = '' | |
| if hint is not None: | |
| prompt += f'Hint: {hint}\n' | |
| prompt += f'{question}\n' | |
| if len(options): | |
| prompt += options_prompt | |
| prompt = 'Study the image carefully and pick the option associated with the correct answer. \ | |
| Focus solely on selecting the option and avoid including any other content.\n' + prompt | |
| message = [dict(type='text', value=prompt)] | |
| message.extend([dict(type='image', value=p) for p in tgt_path]) | |
| return message | |
| def generate_inner(self, message, dataset=None): | |
| prompt, image_path = self.message_to_promptimg(message) | |
| image = Image.open(image_path).convert('RGB') | |
| msgs = [{'role': 'user', 'content': prompt}] | |
| if DATASET_TYPE(dataset) == 'multi-choice': | |
| max_new_tokens = 20 | |
| elif DATASET_TYPE(dataset) == 'Y/N': | |
| max_new_tokens = 100 | |
| else: | |
| max_new_tokens = 1024 | |
| default_kwargs = dict( | |
| max_new_tokens=max_new_tokens, | |
| sampling=False, | |
| num_beams=self.num_beams | |
| ) | |
| default_kwargs.update(self.kwargs) | |
| res, _, _ = self.model.chat( | |
| image=image, | |
| msgs=msgs, | |
| context=None, | |
| tokenizer=self.tokenizer, | |
| **default_kwargs | |
| ) | |
| return res | |