diff --git a/README.md b/README.md index 5e5c5d8a..77cb3c1e 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ ## News 🚀🚀🚀 +- `2024/08/28`: 🚀 We release the [InternVL Omni](https://huggingface.co/OpenGVLab/InternOmni), a model capable of accepting audio input, which maintains good performance in VQA while also showing strong performance in Audio question answering. - `2024/08/01`: The [Chartmimic](https://chartmimic.github.io/) team evaluated the InternVL2 series models on their benchmark. The InternVL2-26B and 76B models achieved the top two performances among open-source models, with the InternVL2 76B model surpassing GeminiProVision and exhibiting comparable results to Claude-3-opus. - `2024/08/01`: InternVL2-Pro achieved the SOTA performance among open-source models on the [CharXiv](https://charxiv.github.io/#leaderboard) dataset, surpassing some well-known closed-source models such as GPT-4V, Gemini 1.5 Flash, and Claude 3 Sonnet. - `2024/07/24`: The [MLVU](https://github.com/JUNJIE99/MLVU) team evaluated InternVL-1.5 on their benchmark. The average performance on the multiple-choice task was 50.4%, while the performance on the generative tasks was 4.02. The performance on the multiple-choice task ranked #1 among all open-source MLLMs. diff --git a/internvl_chat_audio/README.md b/internvl_chat_audio/README.md new file mode 100644 index 00000000..6c8ad75b --- /dev/null +++ b/internvl_chat_audio/README.md @@ -0,0 +1,48 @@ +# InternVL-Chat + +This folder contains the implementation of the InternVL-Chat. + +## 🛠️ Installation + +See [INSTALLATION.md](../INSTALLATION.md) + +In addition, using this codebase requires executing the following steps: + +- Install other requirements: + + ```bash + pip install --upgrade pip # enable PEP 660 support + pip install -e . + ``` + +## 📖 Documents + +- InternVL 2.0 + + - Introduction [\[link\]](https://internvl.readthedocs.io/en/latest/internvl2.0/introduction.html) + - Quick Start [\[link\]](https://internvl.readthedocs.io/en/latest/internvl2.0/quick_start.html) + - Finetune [\[link\]](https://internvl.readthedocs.io/en/latest/internvl2.0/finetune.html) + - Evaluation [\[link\]](https://internvl.readthedocs.io/en/latest/internvl2.0/evaluation.html) + - Deployment [\[link\]](https://internvl.readthedocs.io/en/latest/internvl2.0/deployment.html) + +- InternVL 1.5 + + - Introduction [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.5/introduction.html) + - Quick Start [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.5/quick_start.html) + - Finetune [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.5/finetune.html) + - Evaluation [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.5/evaluation.html) + - Deployment [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.5/deployment.html) + +- InternVL 1.2 + + - Introduction [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.2/introduction.html) + - Quick Start [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.2/quick_start.html) + - Reproduce [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.2/reproduce.html) + - Finetune [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.2/finetune.html) + - Evaluation [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.2/evaluation.html) + +- InternVL 1.1 + + - Introduction [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.1/introduction.html) + - Quick Start [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.1/quick_start.html) + - Evaluation [\[link\]](https://internvl.readthedocs.io/en/latest/internvl1.1/evaluation.html) diff --git a/internvl_chat_audio/eval/caption/evaluate_caption.py b/internvl_chat_audio/eval/caption/evaluate_caption.py new file mode 100644 index 00000000..b9410ef6 --- /dev/null +++ b/internvl_chat_audio/eval/caption/evaluate_caption.py @@ -0,0 +1,272 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from pycocoevalcap.eval import COCOEvalCap +from pycocotools.coco import COCO +from tqdm import tqdm + +ds_collections = { + 'flickr30k': { + 'root': 'data/flickr30k/', + 'annotation': 'data/flickr30k/flickr30k_test_karpathy.json', + 'max_new_tokens': 30, + 'min_new_tokens': 8, + }, + 'coco': { + 'root': 'data/coco/', + 'annotation': ['data/coco/annotations/coco_karpathy_test.json', + 'data/coco/annotations/coco_karpathy_test_gt.json'], + 'max_new_tokens': 30, + 'min_new_tokens': 8, + }, + 'nocaps': { + 'root': 'data/nocaps/images', + 'annotation': 'data/nocaps/nocaps_val_4500_captions.json', + 'max_new_tokens': 30, + 'min_new_tokens': 8, + }, +} + + +class CaptionDataset(torch.utils.data.Dataset): + + def __init__(self, name, root, annotation, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + if name == 'coco': + self.images = json.load(open(annotation)) + else: + self.images = json.load(open(annotation))['images'] + self.name = name + self.prompt = prompt + self.root = root + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + if self.name == 'coco': + filename = self.images[idx]['image'] + image_id = int(filename.split('_')[-1].replace('.jpg', '')) + image_path = os.path.join(self.root, filename) + else: + image_id = self.images[idx]['id'] + if 'file_name' in self.images[idx]: + image_path = os.path.join(self.root, self.images[idx]['file_name']) + else: + image_path = os.path.join(self.root, self.images[idx]['image']) + + image = Image.open(image_path) + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + return { + 'image_id': image_id, + 'input_text': self.prompt, + 'pixel_values': pixel_values + } + + +def collate_fn(inputs, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in inputs], dim=0) + image_ids = [_['image_id'] for _ in inputs] + input_texts = [_['input_text'] for _ in inputs] + input_tokens = tokenizer(input_texts, return_tensors='pt') + + return pixel_values, image_ids, input_tokens.input_ids, input_tokens.attention_mask + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def evaluate_chat_model(): + prompt = 'Provide a one-sentence caption for the provided image.' + print('prompt:', prompt) + random.seed(args.seed) + summaries = [] + + for ds_name in args.datasets: + annotation = ds_collections[ds_name]['annotation'] + if type(annotation) == list: + annotation = annotation[0] + dataset = CaptionDataset( + name=ds_name, + root=ds_collections[ds_name]['root'], + annotation=annotation, + prompt=prompt, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + image_ids, captions = [], [] + for _, (pixel_values, ids, _, _) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=prompt, + generation_config=generation_config, + verbose=True + ) + image_ids.extend(ids) + captions.extend([pred]) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_ids = [None for _ in range(world_size)] + merged_captions = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_ids, image_ids) + torch.distributed.all_gather_object(merged_captions, captions) + + merged_ids = [_ for _ in itertools.chain.from_iterable(merged_ids)] + merged_captions = [_ for _ in itertools.chain.from_iterable(merged_captions)] + average_length = sum(len(x.split()) for x in merged_captions) / len(merged_captions) + print(f'Average caption length: {average_length}') + + if torch.distributed.get_rank() == 0: + print(f'Evaluating {ds_name} ...') + + results = [] + for image_id, caption in zip(merged_ids, merged_captions): + results.append({ + 'image_id': int(image_id), + 'caption': caption, + }) + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + results_file = os.path.join(args.out_dir, results_file) + json.dump(results, open(results_file, 'w')) + + annotation = ds_collections[ds_name]['annotation'] + if type(annotation) == list: + annotation = annotation[-1] + coco = COCO(annotation) + coco_result = coco.loadRes(results_file) + coco_eval = COCOEvalCap(coco, coco_result) + coco_eval.evaluate() + + summary = coco_eval.eval.items() + print(summary) + summaries.append([args.checkpoint, ds_name, average_length, summary]) + + torch.distributed.barrier() + + out_path = '_'.join(args.checkpoint.split('/')[-2:]) + writer = open(os.path.join(args.out_dir, f'{out_path}.txt'), 'a') + print(f"write results to file {os.path.join(args.out_dir, f'{out_path}.txt')}") + for summary in summaries: + print(summary) + writer.write(f'{summary}\n') + writer.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='coco,flickr30k,nocaps') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/cmmmu/evaluate_cmmmu.py b/internvl_chat_audio/eval/cmmmu/evaluate_cmmmu.py new file mode 100644 index 00000000..2af1baff --- /dev/null +++ b/internvl_chat_audio/eval/cmmmu/evaluate_cmmmu.py @@ -0,0 +1,185 @@ +import argparse +import json +import os +import random + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from tqdm import tqdm + +ds_collections = { + 'art_and_design': { + 'root': 'data/', + 'annotation': 'data/cmmmu-data/llava_art_and_design.jsonl', + 'max_new_tokens': 999, + 'min_new_tokens': 1, + }, + 'business': { + 'root': 'data/', + 'annotation': 'data/cmmmu-data/llava_business.jsonl', + 'max_new_tokens': 999, + 'min_new_tokens': 1, + }, + 'health_and_medicine': { + 'root': 'data/', + 'annotation': 'data/cmmmu-data/llava_health_and_medicine.jsonl', + 'max_new_tokens': 999, + 'min_new_tokens': 1, + }, + 'humanities_and_social_sciences': { + 'root': 'data/', + 'annotation': 'data/cmmmu-data/llava_humanities_and_social_sciences.jsonl', + 'max_new_tokens': 999, + 'min_new_tokens': 1, + }, + 'science': { + 'root': 'data/', + 'annotation': 'data/cmmmu-data/llava_science.jsonl', + 'max_new_tokens': 999, + 'min_new_tokens': 1, + }, + 'technology_and_engineering': { + 'root': 'data/', + 'annotation': 'data/cmmmu-data/llava_technology_and_engineering.jsonl', + 'max_new_tokens': 999, + 'min_new_tokens': 1, + }, +} + + +class VQADataset(torch.utils.data.Dataset): + + def __init__(self, root, annotation, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + self.root = root + self.items = [] + f = open(annotation) + data = f.readlines() + for data_line in data: + data_line = json.loads(data_line) + self.items.append(data_line) + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.items) + + def __getitem__(self, idx): + item = self.items[idx] + image_path, question = item['image'], item['text'] + image_path = os.path.join(self.root, image_path) + image = Image.open(image_path).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + return { + 'question': question, + 'pixel_values': pixel_values, + 'item': item, + } + + +def evaluate_chat_model(): + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = VQADataset( + root=ds_collections[ds_name]['root'], + annotation=ds_collections[ds_name]['annotation'], + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + + print(f'Evaluating {ds_name} ...') + results_file = f'{model_id}_{ds_name}.jsonl' + results_file = os.path.join(args.out_dir, results_file) + writer = open(results_file, 'w') + + for _, data in tqdm(enumerate(dataset)): + pixel_value = data['pixel_values'] + question = data['question'] + item = data['item'] + pixel_value = pixel_value.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_value, + question=question, + generation_config=generation_config, + verbose=True + ) + question_id = item['question_id'] + text = item['text'] + output = { + 'question_id': question_id, + 'prompt': text, + 'text': pred, + 'model_id': model_id, + 'metadata': {} + } + writer.write(json.dumps(output, ensure_ascii=False) + '\n') + writer.flush() + print('Results saved to {}'.format(results_file)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=1) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + model_id = '_'.join(args.checkpoint.split('/')[-2:]) + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/llava_bench/eval_gpt_review_bench.py b/internvl_chat_audio/eval/llava_bench/eval_gpt_review_bench.py new file mode 100644 index 00000000..37590a86 --- /dev/null +++ b/internvl_chat_audio/eval/llava_bench/eval_gpt_review_bench.py @@ -0,0 +1,119 @@ +import argparse +import json +import os +import time + +import openai + +NUM_SECONDS_TO_SLEEP = 0.5 + + +def get_eval(content: str, max_tokens: int): + while True: + try: + completion = openai.chat.completions.create( + model='gpt-4-0613', + messages=[{ + 'role': 'system', + 'content': 'You are a helpful and precise assistant for checking the quality of the answer.' + }, { + 'role': 'user', + 'content': content, + }], + temperature=0.2, # TODO: figure out which temperature is best for evaluation + max_tokens=max_tokens, + ) + break + except Exception as e: + print(e) + time.sleep(NUM_SECONDS_TO_SLEEP) + + return completion.choices[0].message.content + + +def parse_score(review): + try: + score_pair = review.split('\n')[0] + score_pair = score_pair.replace(',', ' ') + sp = score_pair.split(' ') + if len(sp) == 2: + return [float(sp[0]), float(sp[1])] + else: + print('error', review) + return [-1, -1] + except Exception as e: + print(e) + print('error', review) + return [-1, -1] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.') + parser.add_argument('-q', '--question') + parser.add_argument('-c', '--context') + parser.add_argument('-a', '--answer-list', nargs='+', default=[]) + parser.add_argument('-r', '--rule') + parser.add_argument('-o', '--output') + parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output') + args = parser.parse_args() + + f_q = open(os.path.expanduser(args.question)) + f_ans1 = open(os.path.expanduser(args.answer_list[0])) + f_ans2 = open(os.path.expanduser(args.answer_list[1])) + rule_dict = json.load(open(os.path.expanduser(args.rule), 'r')) + + if os.path.isfile(os.path.expanduser(args.output)): + cur_reviews = [json.loads(line) for line in open(os.path.expanduser(args.output))] + else: + cur_reviews = [] + + review_file = open(f'{args.output}', 'a') + + context_list = [json.loads(line) for line in open(os.path.expanduser(args.context))] + image_to_context = {context['image']: context for context in context_list} + + handles = [] + idx = 0 + for ques_js, ans1_js, ans2_js in zip(f_q, f_ans1, f_ans2): + ques = json.loads(ques_js) + ans1 = json.loads(ans1_js) + ans2 = json.loads(ans2_js) + print(ques, ans1, ans2) + inst = image_to_context[ques['image']] + + if isinstance(inst['caption'], list): + cap_str = '\n'.join(inst['caption']) + else: + cap_str = inst['caption'] + + category = 'llava_bench_' + json.loads(ques_js)['category'] + if category in rule_dict: + rule = rule_dict[category] + else: + assert False, f'Visual QA category not found in rule file: {category}.' + prompt = rule['prompt'] + role = rule['role'] + content = (f'[Context]\n{cap_str}\n\n' + f'[Question]\n{ques["text"]}\n\n' + f'[{role} 1]\n{ans1["text"]}\n\n[End of {role} 1]\n\n' + f'[{role} 2]\n{ans2["text"]}\n\n[End of {role} 2]\n\n' + f'[System]\n{prompt}\n\n') + cur_js = { + 'id': idx + 1, + 'question_id': ques['question_id'], + 'answer1_id': ans1.get('answer_id', ans1['question_id']), + 'answer2_id': ans2.get('answer_id', ans1['question_id']), + 'category': category + } + if idx >= len(cur_reviews): + review = get_eval(content, args.max_tokens) + scores = parse_score(review) + cur_js['content'] = review + cur_js['tuple'] = scores + review_file.write(json.dumps(cur_js) + '\n') + review_file.flush() + else: + print(f'Skipping {idx} as we already have it.') + idx += 1 + print(idx) + review_file.close() diff --git a/internvl_chat_audio/eval/llava_bench/evaluate_llava_bench.py b/internvl_chat_audio/eval/llava_bench/evaluate_llava_bench.py new file mode 100644 index 00000000..a1b3df18 --- /dev/null +++ b/internvl_chat_audio/eval/llava_bench/evaluate_llava_bench.py @@ -0,0 +1,146 @@ +import argparse +import json +import os +import random + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from tqdm import tqdm + +ds_collections = { + 'llava_bench': { + 'root': 'data/llava-bench-in-the-wild/images', + 'question': 'data/llava-bench-in-the-wild/questions.jsonl', + 'max_new_tokens': 1000, + 'min_new_tokens': 1, + }, +} + + +class VQADataset(torch.utils.data.Dataset): + + def __init__(self, root, data, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + self.root = root + self.data = open(data).readlines() + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + data = json.loads(self.data[idx].strip()) + image, question, question_id, annotation = data['image'], data[ + 'text'], data['question_id'], data.get('answer', None) + + image = os.path.join(self.root, image) + image = Image.open(image).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + question = question + self.prompt + return question_id, question, pixel_values, annotation + + +def evaluate_chat_model(): + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = VQADataset( + root=ds_collections[ds_name]['root'], + data=ds_collections[ds_name]['question'], + prompt=' Please give a detailed answer.', + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + + outputs = [] + for _, (question_id, question, pixel_values, annotations) in tqdm(enumerate(dataset)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=question, + generation_config=generation_config, + verbose=True + ) + outputs.append({ + 'question_id': question_id, + 'text': pred, + 'model_id': model_id, + 'metadata': {} + }) + + print(f'Evaluating {ds_name} ...') + results_file = 'llava_bench_results.jsonl' + results_file = os.path.join(args.out_dir, results_file) + writer = open(results_file, 'w') + for item in outputs: + writer.write(json.dumps(item) + '\n') + writer.close() + print('Results saved to {}'.format(results_file)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='llava_bench') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + model_id = '_'.join(args.checkpoint.split('/')[-2:]) + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/llava_bench/rule.json b/internvl_chat_audio/eval/llava_bench/rule.json new file mode 100644 index 00000000..3c8eafa7 --- /dev/null +++ b/internvl_chat_audio/eval/llava_bench/rule.json @@ -0,0 +1,11 @@ +{ + "coding": {"role": "Assistant", "prompt": "Your task is to evaluate the coding abilities of the above two assistants. They have been asked to implement a program to solve a given problem. Please review their code submissions, paying close attention to their problem-solving approach, code structure, readability, and the inclusion of helpful comments.\n\nPlease ensure that the assistants' submissions:\n\n1. Correctly implement the given problem statement.\n2. Contain accurate and efficient code.\n3. Include clear and concise comments that explain the code's logic and functionality.\n4. Adhere to proper coding standards and best practices.\n\nOnce you have carefully reviewed both submissions, provide detailed feedback on their strengths and weaknesses, along with any suggestions for improvement. You should first output a single line containing two scores on the scale of 1-10 (1: no code/no sense; 10: perfect) for Assistant 1 and 2, respectively. Then give extra comments starting from the next line."}, + "math": {"role": "Assistant", "prompt": "We would like to request your feedback on the mathematical proficiency of two AI assistants regarding the given user question.\nFirstly, please solve the problem independently, without referring to the answers provided by Assistant 1 and Assistant 2.\nAfterward, please examine the problem-solving process of Assistant 1 and Assistant 2 step-by-step to ensure their correctness, identifying any incorrect steps if present. Your evaluation should take into account not only the answer but also the problem-solving steps.\nFinally, please output a Python tuple containing two numerical scores for Assistant 1 and Assistant 2, ranging from 1 to 10, respectively. If applicable, explain the reasons for any variations in their scores and determine which assistant performed better."}, + "default": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, + "conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, + "detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, + "complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, + "llava_bench_conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, + "llava_bench_detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, + "llava_bench_complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."} +} diff --git a/internvl_chat_audio/eval/llava_bench/summarize_gpt_review.py b/internvl_chat_audio/eval/llava_bench/summarize_gpt_review.py new file mode 100644 index 00000000..095d8156 --- /dev/null +++ b/internvl_chat_audio/eval/llava_bench/summarize_gpt_review.py @@ -0,0 +1,62 @@ +import argparse +import json +import os +from collections import defaultdict + +import numpy as np + + +def parse_args(): + parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.') + parser.add_argument('-d', '--dir', default=None) + parser.add_argument('-v', '--version', default=None) + parser.add_argument('-s', '--select', nargs='*', default=None) + parser.add_argument('-f', '--files', nargs='*', default=[]) + parser.add_argument('-i', '--ignore', nargs='*', default=[]) + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + + if args.ignore is not None: + args.ignore = [int(x) for x in args.ignore] + + if len(args.files) > 0: + review_files = args.files + else: + review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and ( + x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith( + 'review_') or 'review' in args.dir)] + + for review_file in sorted(review_files): + config = os.path.basename(review_file).replace('gpt4_text_', '').replace('.jsonl', '') + if args.select is not None and any(x not in config for x in args.select): + continue + if '0613' in config: + version = '0613' + else: + version = '0314' + if args.version is not None and args.version != version: + continue + scores = defaultdict(list) + print(config) + with open(os.path.join(args.dir, review_file) if args.dir is not None else review_file) as f: + for review_str in f: + review = json.loads(review_str) + if review['question_id'] in args.ignore: + continue + if 'category' in review: + scores[review['category']].append(review['tuple']) + scores['all'].append(review['tuple']) + else: + if 'tuple' in review: + scores['all'].append(review['tuple']) + else: + scores['all'].append(review['score']) + for k, v in sorted(scores.items()): + stats = np.asarray(v).mean(0).tolist() + stats = [round(x, 3) for x in stats] + print(k, stats, round(stats[1] / stats[0] * 100, 1)) + print(k, round(stats[1] / stats[0] * 100, 1), round(stats[0] * 10, 1), round(stats[1] * 10, 1)) + print('=================================') diff --git a/internvl_chat_audio/eval/mathvista/calculate_score.py b/internvl_chat_audio/eval/mathvista/calculate_score.py new file mode 100644 index 00000000..3c88df65 --- /dev/null +++ b/internvl_chat_audio/eval/mathvista/calculate_score.py @@ -0,0 +1,258 @@ +import argparse +import sys + +import pandas as pd +# !pip install python-Levenshtein +from Levenshtein import distance +from utilities import * + + +def get_most_similar(prediction, choices): + """ + Use the Levenshtein distance (or edit distance) to determine which of the choices is most similar to the given prediction + """ + distances = [distance(prediction, choice) for choice in choices] + ind = distances.index(min(distances)) + return choices[ind] + # return min(choices, key=lambda choice: distance(prediction, choice)) + + +def normalize_extracted_answer(extraction, choices, question_type, answer_type, precision): + """ + Normalize the extracted answer to match the answer type + """ + if question_type == 'multi_choice': + # make sure the extraction is a string + if isinstance(extraction, str): + extraction = extraction.strip() + else: + try: + extraction = str(extraction) + except: + extraction = '' + + # extract "A" from "(A) text" + letter = re.findall(r'\(([a-zA-Z])\)', extraction) + if len(letter) > 0: + extraction = letter[0].upper() + + options = [chr(ord('A') + i) for i in range(len(choices))] + + if extraction in options: + # convert option letter to text, e.g. "A" -> "text" + ind = options.index(extraction) + extraction = choices[ind] + else: + # select the most similar option + extraction = get_most_similar(extraction, choices) + assert extraction in choices + + elif answer_type == 'integer': + try: + extraction = str(int(float(extraction))) + except: + extraction = None + + elif answer_type == 'float': + try: + extraction = str(round(float(extraction), precision)) + except: + extraction = None + + elif answer_type == 'list': + try: + extraction = str(extraction) + except: + extraction = None + + return extraction + + +def safe_equal(prediction, answer): + """ + Check if the prediction is equal to the answer, even if they are of different types + """ + try: + if prediction == answer: + return True + return False + except Exception as e: + print(e) + return False + + +def get_acc_with_contion(res_pd, key, value): + if key == 'skills': + # if value in res_pd[key]: + total_pd = res_pd[res_pd[key].apply(lambda x: value in x)] + else: + total_pd = res_pd[res_pd[key] == value] + + correct_pd = total_pd[total_pd['true_false'] == True] # noqa: E712 + acc = '{:.2f}'.format(len(correct_pd) / len(total_pd) * 100) + return len(correct_pd), len(total_pd), acc + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--output_dir', type=str, default='./results') + parser.add_argument('--output_file', type=str, default='output.json') + parser.add_argument('--score_file', type=str, default='scores.json') + parser.add_argument('--gt_file', type=str, default='./data/MathVista/annot_testmini.json', help='ground truth file') + parser.add_argument('--number', type=int, default=-1, help='number of problems to run') + parser.add_argument('--rerun', action='store_true', help='rerun the evaluation') + parser.add_argument('--caculate_gain', action='store_true', help='caculate the socre gains over random guess') + parser.add_argument('--random_file', type=str, default='score_random_guess.json') + args = parser.parse_args() + + # args + output_file = os.path.join(args.output_dir, args.output_file) + + # # quick test + # output_file = '../results/llava-llama-2-13b/output_llava_llama_2_13b.json' + + # read json + print(f'Reading {output_file}...') + results = read_json(output_file) + + # read ground truth + print(f'Reading {args.gt_file}...') + gts = read_json(args.gt_file) + + # full pids + full_pids = list(results.keys()) + if args.number > 0: + full_pids = full_pids[:min(args.number, len(full_pids))] + print('Number of testing problems:', len(full_pids)) + + ## [1] Evaluate if the prediction is true or false + print('\nEvaluating the predictions...') + update_json_flag = False + for pid in full_pids: + problem = results[pid] + # print(problem) + + if args.rerun: + if 'prediction' in problem: + del problem['prediction'] + if 'true_false' in problem: + del problem['true_false'] + + choices = problem['choices'] + question_type = problem['question_type'] + answer_type = problem['answer_type'] + precision = problem['precision'] + extraction = problem['extraction'] + + if 'answer' in problem: + answer = problem['answer'] + else: + answer = gts[pid]['answer'] + problem['answer'] = answer + + # normalize the extracted answer to match the answer type + prediction = normalize_extracted_answer(extraction, choices, question_type, answer_type, precision) + + # verify the prediction is true or false + true_false = safe_equal(prediction, answer) + + # update the problem + if 'true_false' not in problem: + update_json_flag = True + + elif true_false != problem['true_false']: + update_json_flag = True + + if 'prediction' not in problem: + update_json_flag = True + + elif prediction != problem['prediction']: + update_json_flag = True + + problem['prediction'] = prediction + problem['true_false'] = true_false + + # save the updated json + if update_json_flag: + print('\n!!!Some problems are updated.!!!') + print(f'\nSaving {output_file}...') + save_json(results, output_file) + + ## [2] Calculate the average accuracy + total = len(full_pids) + correct = 0 + for pid in full_pids: + if results[pid]['true_false']: + correct += 1 + accuracy = str(round(correct / total * 100, 2)) + print(f'\nCorrect: {correct}, Total: {total}, Accuracy: {accuracy}%') + + scores = {'average': {'accuracy': accuracy, 'correct': correct, 'total': total}} + + ## [3] Calculate the fine-grained accuracy scores + + # merge the 'metadata' attribute into the data + for pid in results: + results[pid].update(results[pid].pop('metadata')) + + # convert the data to a pandas DataFrame + df = pd.DataFrame(results).T + + print(len(df)) + print('Number of test problems:', len(df)) + # assert len(df) == 1000 # Important!!! + + # asign the target keys for evaluation + target_keys = ['question_type', 'answer_type', 'language', 'source', 'category', 'task', 'context', 'grade', + 'skills'] + + for key in target_keys: + print(f'\nType: [{key}]') + # get the unique values of the key + if key == 'skills': + # the value is a list + values = [] + for i in range(len(df)): + values += df[key][i] + values = list(set(values)) + else: + values = df[key].unique() + # print(values) + + # calculate the accuracy for each value + scores[key] = {} + for value in values: + correct, total, acc = get_acc_with_contion(df, key, value) + if total > 0: + print(f'[{value}]: {acc}% ({correct}/{total})') + scores[key][value] = {'accuracy': acc, 'correct': correct, 'total': total} + + # sort the scores by accuracy + scores[key] = dict(sorted(scores[key].items(), key=lambda item: float(item[1]['accuracy']), reverse=True)) + + # save the scores + scores_file = os.path.join(args.output_dir, args.score_file) + print(f'\nSaving {scores_file}...') + save_json(scores, scores_file) + print('\nDone!') + + # [4] Calculate the score gains over random guess + if args.caculate_gain: + random_file = os.path.join(args.output_dir, args.random_file) + random_scores = json.load(open(random_file)) + + print('\nCalculating the score gains...') + for key in scores: + if key == 'average': + gain = round(float(scores[key]['accuracy']) - float(random_scores[key]['accuracy']), 2) + scores[key]['acc_gain'] = gain + else: + for sub_key in scores[key]: + gain = round( + float(scores[key][sub_key]['accuracy']) - float(random_scores[key][sub_key]['accuracy']), 2) + scores[key][sub_key]['acc_gain'] = str(gain) + + # save the score gains + print(f'\nSaving {scores_file}...') + save_json(scores, scores_file) + print('\nDone!') diff --git a/internvl_chat_audio/eval/mathvista/evaluate_mathvista.py b/internvl_chat_audio/eval/mathvista/evaluate_mathvista.py new file mode 100644 index 00000000..d7bcd5a7 --- /dev/null +++ b/internvl_chat_audio/eval/mathvista/evaluate_mathvista.py @@ -0,0 +1,222 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from datasets import concatenate_datasets, load_dataset +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from torch.utils.data import Dataset +from tqdm import tqdm + +ds_collections = { + 'MathVista_testmini': { + 'root': 'AI4Math/MathVista', + 'max_new_tokens': 1000, + 'min_new_tokens': 1, + 'split': 'testmini' + }, + 'MathVista_test': { + 'root': 'AI4Math/MathVista', + 'max_new_tokens': 1000, + 'min_new_tokens': 1, + 'split': 'test' + }, +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + data_items = [_['data_item'] for _ in batches] + return pixel_values, data_items + + +class MathVistaDataset(torch.utils.data.Dataset): + + def __init__(self, root, split, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + dataset = load_dataset(root, cache_dir=os.path.join(os.getcwd(), 'data/MathVista/')) + self.data = dataset[split] + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + data_item = self.data[idx] + image = data_item['decoded_image'] + del data_item['decoded_image'] + + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + return { + 'pixel_values': pixel_values, + 'data_item': data_item, + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def evaluate_chat_model(): + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = MathVistaDataset( + root=ds_collections[ds_name]['root'], + split=ds_collections[ds_name]['split'], + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, data_items) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=data_items[0]['query'], + generation_config=generation_config, + verbose=True + ) + + data_item = data_items[0] + data_item['response'] = pred + outputs.append(data_item) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + temp = {} + for data_item in merged_outputs: + pid = data_item['pid'] + temp[pid] = data_item + + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + output_path = os.path.join(args.out_dir, results_file) + json.dump(temp, open(output_path, 'w'), indent=4) + print('Results saved to {}'.format(output_path)) + + cmd = f'python eval/mathvista/extract_answer.py --output_file {results_file}' + print(cmd) + os.system(cmd) + + cmd = f'python eval/mathvista/calculate_score.py --output_file {results_file} --score_file {results_file[:-5]}_score.json' + print(cmd) + os.system(cmd) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='MathVista_testmini') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/mathvista/extract_answer.py b/internvl_chat_audio/eval/mathvista/extract_answer.py new file mode 100644 index 00000000..8e37b92e --- /dev/null +++ b/internvl_chat_audio/eval/mathvista/extract_answer.py @@ -0,0 +1,142 @@ +import argparse + +# OpenAI +import openai +from tqdm import tqdm +from utilities import * + +openai.api_key = os.getenv('OPENAI_API_KEY') +print(openai.api_key) + +# load demo prompt +from prompts.ext_ans import demo_prompt + + +def verify_extraction(extraction): + extraction = extraction.strip() + if extraction == '' or extraction is None: + return False + return True + + +def create_test_prompt(demo_prompt, query, response): + demo_prompt = demo_prompt.strip() + test_prompt = f'{query}\n\n{response}' + full_prompt = f'{demo_prompt}\n\n{test_prompt}\n\nExtracted answer: ' + return full_prompt + + +def extract_answer(response, problem, quick_extract=False): + question_type = problem['question_type'] + answer_type = problem['answer_type'] + choices = problem['choices'] + query = problem['query'] + + if response == '': + return '' + + if question_type == 'multi_choice' and response in choices: + return response + + if answer_type == 'integer': + try: + extraction = int(response) + return str(extraction) + except: + pass + + if answer_type == 'float': + try: + extraction = str(float(response)) + return extraction + except: + pass + + # quick extraction + if quick_extract: + print('Quickly extracting answer...') + # The answer is "text". -> "text" + try: + result = re.search(r'The answer is "(.*)"\.', response) + if result: + extraction = result.group(1) + return extraction + except: + pass + + # general extraction + try: + full_prompt = create_test_prompt(demo_prompt, query, response) + extraction = get_chat_response(full_prompt, openai.api_key, patience=5) + return extraction + except Exception as e: + print(e) + print(f'Error in extracting answer for {pid}') + + return '' + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + # input + parser.add_argument('--output_dir', type=str, default='./results') + parser.add_argument('--output_file', type=str, default='mathvista_answer.json') + parser.add_argument('--response_label', type=str, default='response', help='response label for the input file') + # model + parser.add_argument('--llm_engine', type=str, default='gpt-4-0613', help='llm engine', + choices=['gpt-3.5-turbo', 'gpt-3.5', 'gpt-4', 'gpt-4-0314', 'gpt-4-0613']) + parser.add_argument('--number', type=int, default=-1, help='number of problems to run') + parser.add_argument('--quick_extract', action='store_true', help='use rules to extract answer for some problems') + parser.add_argument('--rerun', action='store_true', help='rerun the answer extraction') + # output + parser.add_argument('--save_every', type=int, default=10, help='save every n problems') + parser.add_argument('--output_label', type=str, default='', help='label for the output file') + args = parser.parse_args() + + # args + label = args.response_label + result_file = os.path.join(args.output_dir, args.output_file) + + if args.output_label != '': + output_file = result_file.replace('.json', f'_{args.output_label}.json') + else: + output_file = result_file + + # read results + print(f'Reading {result_file}...') + results = read_json(result_file) + + # full pids + full_pids = list(results.keys()) + if args.number > 0: + full_pids = full_pids[:min(args.number, len(full_pids))] + print('Number of testing problems:', len(full_pids)) + + # test pids + if args.rerun: + test_pids = full_pids + else: + test_pids = [] + for pid in full_pids: + # print(pid) + if 'extraction' not in results[pid] or not verify_extraction(results[pid]['extraction']): + test_pids.append(pid) + + test_num = len(test_pids) + print('Number of problems to run:', test_num) + # print(test_pids) + + # tqdm, enumerate results + for i, pid in enumerate(tqdm(test_pids)): + problem = results[pid] + + assert label in problem + response = problem[label] + + extraction = extract_answer(response, problem, args.quick_extract) + results[pid]['extraction'] = extraction + + if i % args.save_every == 0 or i == test_num - 1: + print(f'Saving results to {output_file}...') + save_json(results, output_file) + print(f'Results saved.') diff --git a/internvl_chat_audio/eval/mathvista/prompts/ext_ans.py b/internvl_chat_audio/eval/mathvista/prompts/ext_ans.py new file mode 100644 index 00000000..0a787ebe --- /dev/null +++ b/internvl_chat_audio/eval/mathvista/prompts/ext_ans.py @@ -0,0 +1,42 @@ + + +# pids = 852, 104, 824, 506, 540 + +demo_prompt = """ +Please read the following example. Then extract the answer from the model response and type it at the end of the prompt. + +Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end. +Question: Which number is missing? + +Model response: The number missing in the sequence is 14. + +Extracted answer: 14 + +Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end. +Question: What is the fraction of females facing the camera? + +Model response: The fraction of females facing the camera is 0.6, which means that six out of ten females in the group are facing the camera. + +Extracted answer: 0.6 + +Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end. +Question: How much money does Luca need to buy a sour apple candy and a butterscotch candy? (Unit: $) + +Model response: Luca needs $1.45 to buy a sour apple candy and a butterscotch candy. + +Extracted answer: 1.45 + +Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end. +Question: Between which two years does the line graph saw its maximum peak? + +Model response: The line graph saw its maximum peak between 2007 and 2008. + +Extracted answer: [2007, 2008] + +Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end. +Question: What fraction of the shape is blue?\nChoices:\n(A) 3/11\n(B) 8/11\n(C) 6/11\n(D) 3/5 + +Model response: The correct answer is (B) 8/11. + +Extracted answer: B +""" diff --git a/internvl_chat_audio/eval/mathvista/utilities.py b/internvl_chat_audio/eval/mathvista/utilities.py new file mode 100644 index 00000000..9f23f0f6 --- /dev/null +++ b/internvl_chat_audio/eval/mathvista/utilities.py @@ -0,0 +1,202 @@ +import json +import os +import pickle +import re +import time + +import cv2 +import openai +from word2number import w2n + + +def create_dir(output_dir): + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + +def read_csv(file): + data = [] + with open(file, 'r') as f: + for line in f: + data.append(line.strip()) + return data + + +def read_pandas_csv(csv_path): + # read a pandas csv sheet + import pandas as pd + df = pd.read_csv(csv_path) + return df + + +def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def read_jsonl(file): + with open(file, 'r') as f: + data = [json.loads(line) for line in f] + return data + + +def read_pickle(path): + with open(path, 'rb') as f: + return pickle.load(f) + + +def save_json(data, path): + with open(path, 'w') as f: + json.dump(data, f, indent=4) + + +def save_array_img(path, image): + cv2.imwrite(path, image) + + +def contains_digit(text): + # check if text contains a digit + if any(char.isdigit() for char in text): + return True + return False + + +def contains_number_word(text): + # check if text contains a number word + ignore_words = ['a', 'an', 'point'] + words = re.findall(r'\b\w+\b', text) # This regex pattern matches any word in the text + for word in words: + if word in ignore_words: + continue + try: + w2n.word_to_num(word) + return True # If the word can be converted to a number, return True + except ValueError: + continue # If the word can't be converted to a number, continue with the next word + + # check if text contains a digit + if any(char.isdigit() for char in text): + return True + + return False # If none of the words could be converted to a number, return False + + +def contains_quantity_word(text, special_keep_words=[]): + # check if text contains a quantity word + quantity_words = ['most', 'least', 'fewest' + 'more', 'less', 'fewer', + 'largest', 'smallest', 'greatest', + 'larger', 'smaller', 'greater', + 'highest', 'lowest', 'higher', 'lower', + 'increase', 'decrease', + 'minimum', 'maximum', 'max', 'min', + 'mean', 'average', 'median', + 'total', 'sum', 'add', 'subtract', + 'difference', 'quotient', 'gap', + 'half', 'double', 'twice', 'triple', + 'square', 'cube', 'root', + 'approximate', 'approximation', + 'triangle', 'rectangle', 'circle', 'square', 'cube', 'sphere', 'cylinder', 'cone', 'pyramid', + 'multiply', 'divide', + 'percentage', 'percent', 'ratio', 'proportion', 'fraction', 'rate', + ] + + quantity_words += special_keep_words # dataset specific words + + words = re.findall(r'\b\w+\b', text) # This regex pattern matches any word in the text + if any(word in quantity_words for word in words): + return True + + return False # If none of the words could be converted to a number, return False + + +def is_bool_word(text): + if text in ['Yes', 'No', 'True', 'False', + 'yes', 'no', 'true', 'false', + 'YES', 'NO', 'TRUE', 'FALSE']: + return True + return False + + +def is_digit_string(text): + # remove ".0000" + text = text.strip() + text = re.sub(r'\.0+$', '', text) + try: + int(text) + return True + except ValueError: + return False + + +def is_float_string(text): + # text is a float string if it contains a "." and can be converted to a float + if '.' in text: + try: + float(text) + return True + except ValueError: + return False + return False + + +def copy_image(image_path, output_image_path): + from shutil import copyfile + copyfile(image_path, output_image_path) + + +def copy_dir(src_dir, dst_dir): + from shutil import copytree + + # copy the source directory to the target directory + copytree(src_dir, dst_dir) + + +import PIL.Image as Image + + +def get_image_size(img_path): + img = Image.open(img_path) + width, height = img.size + return width, height + + +def get_chat_response(promot, api_key, model='gpt-3.5-turbo', temperature=0, max_tokens=256, n=1, patience=10000000, + sleep_time=0): + messages = [ + {'role': 'user', 'content': promot}, + ] + while patience > 0: + patience -= 1 + try: + completion = openai.chat.completions.create(model=model, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + n=n) + if n == 1: + prediction = completion.choices[0].message.content.strip() + if prediction != '' and prediction is not None: + return prediction + else: + prediction = [choice.message.content.strip() for choice in completion.choices] + if prediction[0] != '' and prediction[0] is not None: + return prediction + + except Exception as e: + if 'Rate limit' not in str(e): + print(e) + + if 'Please reduce the length of the messages' in str(e): + print('!!Reduce promot size') + # reduce input prompt and keep the tail + new_size = int(len(promot) * 0.9) + new_start = len(promot) - new_size + promot = promot[new_start:] + messages = [ + {'role': 'user', 'content': promot}, + ] + + if sleep_time > 0: + time.sleep(sleep_time) + return '' diff --git a/internvl_chat_audio/eval/mmbench/evaluate_mmbench.py b/internvl_chat_audio/eval/mmbench/evaluate_mmbench.py new file mode 100644 index 00000000..ea4f806b --- /dev/null +++ b/internvl_chat_audio/eval/mmbench/evaluate_mmbench.py @@ -0,0 +1,313 @@ +import argparse +import base64 +import itertools +import json +import os +import random +import time +from functools import partial +from io import BytesIO + +import pandas as pd +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from torch.utils.data import Dataset +from tqdm import tqdm + +ds_collections = { + 'mmbench_dev_20230712': { + 'root': 'data/mmbench/mmbench_dev_20230712.tsv', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + 'type': 'dev', + 'language': 'en' + }, + 'mmbench_dev_cn_20231003': { + 'root': 'data/mmbench/mmbench_dev_cn_20231003.tsv', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + 'type': 'dev', + 'language': 'cn' + }, + 'mmbench_dev_en_20231003': { + 'root': 'data/mmbench/mmbench_dev_en_20231003.tsv', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + 'type': 'dev', + 'language': 'en' + }, + 'mmbench_test_cn_20231003': { + 'root': 'data/mmbench/mmbench_test_cn_20231003.tsv', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + 'type': 'test', + 'language': 'cn' + }, + 'mmbench_test_en_20231003': { + 'root': 'data/mmbench/mmbench_test_en_20231003.tsv', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + 'type': 'test', + 'language': 'en' + }, + 'ccbench_dev_cn': { + 'root': 'data/mmbench/CCBench_legacy.tsv', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + 'type': 'dev', + 'language': 'cn' + } +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + answers = [_['answer'] for _ in batches] + indexes = [_['index'] for _ in batches] + options = [_['option'] for _ in batches] + return pixel_values, questions, answers, indexes, options + + +class MMBenchDataset(torch.utils.data.Dataset): + + def __init__(self, root, prompt, language, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + self.df = pd.read_csv(root, sep='\t') + self.prompt = prompt + self.language = language + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.df) + + def __getitem__(self, idx): + index = self.df.iloc[idx]['index'] + image = self.df.iloc[idx]['image'] + question = self.df.iloc[idx]['question'] + answer = self.df.iloc[idx]['answer'] if 'answer' in self.df.iloc[0].keys() else None + # catetory = self.df.iloc[idx]['category'] + # l2_catetory = self.df.iloc[idx]['l2-category'] + + image = Image.open(BytesIO(base64.b64decode(image))).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + option_candidate = ['A', 'B', 'C', 'D', 'E'] + options = { + cand: self.load_from_df(idx, cand) + for cand in option_candidate + if self.load_from_df(idx, cand) is not None + } + + hint = self.load_from_df(idx, 'hint') + if hint is not None: + question = hint + '\n' + question + for key, item in options.items(): + question += f'\n{key}. {item}' + if self.language == 'cn': + question = question + '\n' + self.prompt['cn'] + else: + question = question + '\n' + self.prompt['en'] + + return { + 'question': question, + 'pixel_values': pixel_values, + 'answer': answer, + 'index': index, + 'option': options + } + + def load_from_df(self, idx, key): + if key in self.df.iloc[idx] and not pd.isna(self.df.iloc[idx][key]): + return self.df.iloc[idx][key] + else: + return None + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def post_process(pred, option): + pred = pred.strip() + option_candidate = list(option.keys()) + if len(pred) == 1: + return pred + elif len(pred) != 1 and pred[0] in option_candidate: + return pred[0] + elif len(pred) != 1 and pred[0] not in option_candidate: + for k, v in option.items(): + if v in pred: + return k + + return pred + + +def evaluate_chat_model(): + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = MMBenchDataset( + root=ds_collections[ds_name]['root'], + prompt=prompt, + language=ds_collections[ds_name]['language'], + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, answers, indexes, options) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + preds = [post_process(pred, options[0])] + + for question, pred, answer, index in zip(questions, preds, answers, indexes): + outputs.append({ + 'question': question, + 'answer': pred, + 'gt_answers': answer, + 'index': int(index) + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.xlsx' + output_path = os.path.join(args.out_dir, results_file) + df = pd.read_table(ds_collections[ds_name]['root']) + cur_df = df.copy() + if 'mmbench' in ds_name: + cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category']) + cur_df.insert(6, 'prediction', None) + else: + cur_df = cur_df.drop(columns=['category', 'image']) + cur_df.insert(8, 'prediction', None) + for item in merged_outputs: + cur_df.loc[df['index'] == item['index'], 'prediction'] = item['answer'] + + cur_df.to_excel(output_path, index=False, engine='openpyxl') + print('Results saved to {}'.format(output_path)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='mmbench_dev_20230712') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + prompt = { + 'en': "Answer with the option's letter from the given choices directly.", + 'cn': '请直接回答选项字母。' + } + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/mme/README.md b/internvl_chat_audio/eval/mme/README.md new file mode 100644 index 00000000..f265dac8 --- /dev/null +++ b/internvl_chat_audio/eval/mme/README.md @@ -0,0 +1,16 @@ +# This is an automated calculation script for the acc, acc+, and score. + +# You can directly run "python3 calculation.py" to get the evaluation results of LaVIN. + +# In order to get the statistical results of your model: + +(1) Fill all the files in "Your_Results", adding your model's responses: +Each file in "Your_Results" consists of: +Image_Name + "\\t" + Question + "\\t" + Ground_Truth_Answer + "\\n" + +You need to add the responses of your model as: +Image_Name + "\\t" + Question + "\\t" + Ground_Truth_Answer + "\\t" + Your_Response + "\\n" + +Note: if your responses contain "\\n", please delet it. For each question, your response can only be in one line, not across lines! + +(2) run "python3 calculation.py --results_dir ./Your_Results" diff --git a/internvl_chat_audio/eval/mme/Your_Results/OCR.txt b/internvl_chat_audio/eval/mme/Your_Results/OCR.txt new file mode 100644 index 00000000..7516e67f --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/OCR.txt @@ -0,0 +1,40 @@ +0001.jpg Is the word in the logo "angie's"? Please answer yes or no. Yes +0001.jpg Is the word in the logo "angle's"? Please answer yes or no. No +0002.jpg Is the word in the logo "c'est cheese"? Please answer yes or no. Yes +0002.jpg Is the word in the logo "crest cheese"? Please answer yes or no. No +0003.jpg Is the word in the logo "beavertails pastry"? Please answer yes or no. Yes +0003.jpg Is the word in the logo "beavertalls pastry"? Please answer yes or no. No +0004.jpg Is the word in the logo "old market sundries"? Please answer yes or no. Yes +0004.jpg Is the word in the logo "old market hundreds"? Please answer yes or no. No +0005.jpg Is the word in the logo "kress"? Please answer yes or no. Yes +0005.jpg Is the word in the logo "dress"? Please answer yes or no. No +0006.jpg Is the word in the logo "the beatles story liver pool"? Please answer yes or no. Yes +0006.jpg Is the word in the logo "the beats story liver pool"? Please answer yes or no. No +0007.jpg Is the phone number in the picture "0131 555 6363"? Please answer yes or no. Yes +0007.jpg Is the phone number in the picture "0137 556 6363"? Please answer yes or no. No +0008.jpg Is the word in the logo "phil's market"? Please answer yes or no. Yes +0008.jpg Is the word in the logo "phll's market"? Please answer yes or no. No +0009.jpg Is the word in the logo "fenders diner"? Please answer yes or no. Yes +0009.jpg Is the word in the logo "finders diner"? Please answer yes or no. No +0010.jpg Is the word in the logo "high time coffee shop"? Please answer yes or no. Yes +0010.jpg Is the word in the logo "high tite cofeee shop"? Please answer yes or no. No +0011.jpg Is the word in the logo "ihop restaurant"? Please answer yes or no. Yes +0011.jpg Is the word in the logo "lhop restaurant"? Please answer yes or no. No +0012.jpg Is the word in the logo "casa grecque restaurants"? Please answer yes or no. Yes +0012.jpg Is the word in the logo "case grecque restaurants"? Please answer yes or no. No +0013.jpg Is the word in the picture "seabreeze motel"? Please answer yes or no. Yes +0013.jpg Is the word in the picture "seebreeze model"? Please answer yes or no. No +0014.jpg Is the word in the logo "penarth pier built 1894"? Please answer yes or no. Yes +0014.jpg Is the word in the logo "penarth pies buid 1894"? Please answer yes or no. No +0015.jpg Is the text in the picture "hollywood"? Please answer yes or no. Yes +0015.jpg Is the text in the picture "holly word"? Please answer yes or no. No +0016.jpg Is the word in the logo "shop rite"? Please answer yes or no. Yes +0016.jpg Is the word in the logo "stop rite"? Please answer yes or no. No +0017.jpg Is the word in the logo "hardco industrial construction"? Please answer yes or no. Yes +0017.jpg Is the word in the logo "hardto industal construction"? Please answer yes or no. No +0018.jpg Is the word in the logo "oldsmobile service"? Please answer yes or no. Yes +0018.jpg Is the word in the logo "old mobile service"? Please answer yes or no. No +0019.jpg Is the word in the logo "exchange hotel"? Please answer yes or no. Yes +0019.jpg Is the word in the logo "excharge hotel"? Please answer yes or no. No +0020.jpg Is the word in the logo "cold drinks"? Please answer yes or no. Yes +0020.jpg Is the word in the logo "cold rinks"? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/artwork.txt b/internvl_chat_audio/eval/mme/Your_Results/artwork.txt new file mode 100644 index 00000000..4ff2d5e1 --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/artwork.txt @@ -0,0 +1,400 @@ +10002.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +10002.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No +10049.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +10049.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. No +10256.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +10256.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. No +10358.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +10358.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No +10543.jpg Is this artwork displayed in fogg art museum, harvard university, cambridge? Please answer yes or no. Yes +10543.jpg Is this artwork displayed in museo civico, pistoia? Please answer yes or no. No +10581.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes +10581.jpg Does this artwork belong to the type of genre? Please answer yes or no. No +1060.jpg Is this artwork created by antoniazzo romano? Please answer yes or no. Yes +1060.jpg Is this artwork created by gentile da fabriano? Please answer yes or no. No +10881.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +10881.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No +10970.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +10970.jpg Does this artwork belong to the type of study? Please answer yes or no. No +11276.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes +11276.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No +11331.jpg Is this artwork created by donatello? Please answer yes or no. Yes +11331.jpg Is this artwork created by zichy, mihály? Please answer yes or no. No +11488.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes +11488.jpg Does this artwork belong to the type of historical? Please answer yes or no. No +11724.jpg Is this artwork created by duccio di buoninsegna? Please answer yes or no. Yes +11724.jpg Is this artwork created by giani, felice? Please answer yes or no. No +11726.jpg Is this artwork titled temptation on the mountain (detail)? Please answer yes or no. Yes +11726.jpg Is this artwork titled in the forest of fontainebleau? Please answer yes or no. No +12133.jpg Is this artwork titled hand study with bible? Please answer yes or no. Yes +12133.jpg Is this artwork titled self-portrait aged 78? Please answer yes or no. No +12439.jpg Is this artwork created by dürer, albrecht? Please answer yes or no. Yes +12439.jpg Is this artwork created by koekkoek, barend cornelis? Please answer yes or no. No +12561.jpg Is this artwork created by eberlein, gustav heinrich? Please answer yes or no. Yes +12561.jpg Is this artwork created by gillemans, jan pauwel the younger? Please answer yes or no. No +12652.jpg Is this artwork displayed in stedelijk museum de lakenhal, leiden? Please answer yes or no. Yes +12652.jpg Is this artwork displayed in palazzo ducale, mantua? Please answer yes or no. No +12736.jpg Is this artwork displayed in cannon hall museum, barnsley? Please answer yes or no. Yes +12736.jpg Is this artwork displayed in protestant parish church, gelnhausen? Please answer yes or no. No +12902.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +12902.jpg Is this artwork displayed in musée national gustave-moreau, paris? Please answer yes or no. No +12908.jpg Is this artwork titled ruth and boaz? Please answer yes or no. Yes +12908.jpg Is this artwork titled view of dresden from the right bank of the elbe with the augustus bridge? Please answer yes or no. No +13091.jpg Is this artwork titled italianate landscape with figures by classical ruins? Please answer yes or no. Yes +13091.jpg Is this artwork titled two boys singing? Please answer yes or no. No +13174.jpg Is this artwork titled nobility? Please answer yes or no. Yes +13174.jpg Is this artwork titled aretino in the studio of tintoretto? Please answer yes or no. No +13239.jpg Is this artwork titled doge ziani receiving the benediction of pope alexander iii? Please answer yes or no. Yes +13239.jpg Is this artwork titled the adoration of the shepherds? Please answer yes or no. No +13288.jpg Does this artwork exist in the form of architecture? Please answer yes or no. Yes +13288.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +13696.jpg Is this artwork displayed in pinacoteca nazionale, siena? Please answer yes or no. Yes +13696.jpg Is this artwork displayed in british embassy, paris? Please answer yes or no. No +13760.jpg Is this artwork titled noli me tangere? Please answer yes or no. Yes +13760.jpg Is this artwork titled profile study of a bearded man? Please answer yes or no. No +13821.jpg Is this artwork created by frangipane, niccolò? Please answer yes or no. Yes +13821.jpg Is this artwork created by drevet, pierre? Please answer yes or no. No +13901.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +13901.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +14283.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +14283.jpg Does this artwork exist in the form of mosaic? Please answer yes or no. No +14499.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +14499.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No +14777.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +14777.jpg Does this artwork belong to the type of historical? Please answer yes or no. No +15028.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes +15028.jpg Does this artwork belong to the type of study? Please answer yes or no. No +15232.jpg Is this artwork created by giordano, luca? Please answer yes or no. Yes +15232.jpg Is this artwork created by heyerdahl, hans olaf? Please answer yes or no. No +15246.jpg Is this artwork displayed in palazzo medici riccardi, florence? Please answer yes or no. Yes +15246.jpg Is this artwork displayed in abbey church of sainte-foy, conques (aveyron)? Please answer yes or no. No +15311.jpg Is this artwork created by giorgione? Please answer yes or no. Yes +15311.jpg Is this artwork created by marilhat, prosper? Please answer yes or no. No +15989.jpg Is this artwork displayed in pinacoteca, vatican? Please answer yes or no. Yes +15989.jpg Is this artwork displayed in cathedral museum, zamora? Please answer yes or no. No +16006.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +16006.jpg Is this artwork displayed in cathedral of san geminiano, modena? Please answer yes or no. No +16249.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes +16249.jpg Does this artwork belong to the type of religious? Please answer yes or no. No +16538.jpg Is this artwork created by gogh, vincent van? Please answer yes or no. Yes +16538.jpg Is this artwork created by altdorfer, albrecht? Please answer yes or no. No +16835.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +16835.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No +16911.jpg Is this artwork created by gossart, jan? Please answer yes or no. Yes +16911.jpg Is this artwork created by stanzione, massimo? Please answer yes or no. No +17311.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +17311.jpg Does this artwork belong to the type of interior? Please answer yes or no. No +17317.jpg Is this artwork created by gozzoli, benozzo? Please answer yes or no. Yes +17317.jpg Is this artwork created by coriolano, cristoforo? Please answer yes or no. No +17535.jpg Is this artwork created by grebber, pieter de? Please answer yes or no. Yes +17535.jpg Is this artwork created by massys, quentin? Please answer yes or no. No +17823.jpg Is this artwork created by greuze, jean-baptiste? Please answer yes or no. Yes +17823.jpg Is this artwork created by landseer, sir edwin henry? Please answer yes or no. No +17838.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +17838.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No +17998.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +17998.jpg Does this artwork belong to the type of genre? Please answer yes or no. No +18566.jpg Is this artwork created by hamen, juan van der? Please answer yes or no. Yes +18566.jpg Is this artwork created by starnina, gherardo di jacopo? Please answer yes or no. No +18604.jpg Is this artwork created by hardouin-mansart, jules? Please answer yes or no. Yes +18604.jpg Is this artwork created by kerseboom, friedrich? Please answer yes or no. No +18722.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +18722.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. No +1873.jpg Does this artwork exist in the form of architecture? Please answer yes or no. Yes +1873.jpg Does this artwork exist in the form of painting? Please answer yes or no. No +18902.jpg Is this artwork created by herrera, francisco de, the elder? Please answer yes or no. Yes +18902.jpg Is this artwork created by ingres, jean-auguste-dominique? Please answer yes or no. No +18926.jpg Is this artwork created by herring, john frederick the younger? Please answer yes or no. Yes +18926.jpg Is this artwork created by cozens, john robert? Please answer yes or no. No +19087.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +19087.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +19154.jpg Is this artwork titled portrait of the merchant georg gisze (detail)? Please answer yes or no. Yes +19154.jpg Is this artwork titled pair of table candlesticks? Please answer yes or no. No +19417.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +19417.jpg Does this artwork exist in the form of mosaic? Please answer yes or no. No +19452.jpg Is this artwork titled the artist and his model? Please answer yes or no. Yes +19452.jpg Is this artwork titled the lovesick maiden (detail)? Please answer yes or no. No +19839.jpg Is this artwork created by janneck, franz christoph? Please answer yes or no. Yes +19839.jpg Is this artwork created by goupil, jules-adolphe? Please answer yes or no. No +19863.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +19863.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No +19993.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +19993.jpg Is this artwork displayed in cathedral of st paul, liège? Please answer yes or no. No +20176.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +20176.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No +20437.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +20437.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No +20442.jpg Is this artwork created by kucharski, aleksander? Please answer yes or no. Yes +20442.jpg Is this artwork created by pourbus, frans the elder? Please answer yes or no. No +20455.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +20455.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +20483.jpg Is this artwork titled allegory of the regency? Please answer yes or no. Yes +20483.jpg Is this artwork titled breton woman bathing? Please answer yes or no. No +20490.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +20490.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No +20551.jpg Is this artwork created by lagrenée, jean-jacques? Please answer yes or no. Yes +20551.jpg Is this artwork created by scultori, diana? Please answer yes or no. No +20651.jpg Is this artwork titled a highland landscape? Please answer yes or no. Yes +20651.jpg Is this artwork titled a dog and a cat fighting in a kitchen interior? Please answer yes or no. No +20724.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes +20724.jpg Does this artwork belong to the type of landscape? Please answer yes or no. No +21048.jpg Is this artwork created by lemoyne, jean-baptiste ii? Please answer yes or no. Yes +21048.jpg Is this artwork created by kneller, sir godfrey? Please answer yes or no. No +21097.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +21097.jpg Does this artwork belong to the type of genre? Please answer yes or no. No +21244.jpg Does this artwork belong to the type of study? Please answer yes or no. Yes +21244.jpg Does this artwork belong to the type of portrait? Please answer yes or no. No +21469.jpg Does this artwork belong to the type of genre? Please answer yes or no. Yes +21469.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No +21580.jpg Is this artwork created by linard, jacques? Please answer yes or no. Yes +21580.jpg Is this artwork created by bonino da campione? Please answer yes or no. No +21712.jpg Is this artwork titled st john the evangelist resuscitating drusiana? Please answer yes or no. Yes +21712.jpg Is this artwork titled la finette? Please answer yes or no. No +22329.jpg Is this artwork titled marriage of the virgin? Please answer yes or no. Yes +22329.jpg Is this artwork titled landscape with river and figures (detail)? Please answer yes or no. No +22366.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +22366.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No +22667.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +22667.jpg Is this artwork displayed in san francesco d'assisi, pavia? Please answer yes or no. No +22760.jpg Is this artwork titled madonna and child (detail)? Please answer yes or no. Yes +22760.jpg Is this artwork titled view of the south and east walls? Please answer yes or no. No +22842.jpg Is this artwork titled ukrainian peasant girl? Please answer yes or no. Yes +22842.jpg Is this artwork titled virtue crowning merit? Please answer yes or no. No +23229.jpg Is this artwork displayed in national gallery, london? Please answer yes or no. Yes +23229.jpg Is this artwork displayed in notre-dame-la-riche, tours? Please answer yes or no. No +23427.jpg Is this artwork displayed in the hermitage, st. petersburg? Please answer yes or no. Yes +23427.jpg Is this artwork displayed in national gallery of victoria, melbourne? Please answer yes or no. No +23465.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +23465.jpg Is this artwork displayed in cistertian church, zirc? Please answer yes or no. No +23824.jpg Is this artwork titled christ walking on the water? Please answer yes or no. Yes +23824.jpg Is this artwork titled mademoiselle romaine lacaux? Please answer yes or no. No +24122.jpg Is this artwork displayed in museo correr, venice? Please answer yes or no. Yes +24122.jpg Is this artwork displayed in church of brou, bourg-en-bresse? Please answer yes or no. No +24260.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +24260.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No +24291.jpg Is this artwork titled virgin and child with sts catherine, cecilia, barbara, and ursula? Please answer yes or no. Yes +24291.jpg Is this artwork titled sorrow? Please answer yes or no. No +24723.jpg Is this artwork titled tomb of henry the lion and his wife matilda? Please answer yes or no. Yes +24723.jpg Is this artwork titled god the father? Please answer yes or no. No +2490.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes +2490.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No +2507.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +2507.jpg Is this artwork displayed in st. vitus's cathedral, prague? Please answer yes or no. No +25312.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +25312.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +25476.jpg Is this artwork created by michelangelo buonarroti? Please answer yes or no. Yes +25476.jpg Is this artwork created by beuckelaer, joachim? Please answer yes or no. No +25492.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes +25492.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No +25513.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +25513.jpg Does this artwork belong to the type of landscape? Please answer yes or no. No +26521.jpg Does this artwork exist in the form of illumination? Please answer yes or no. Yes +26521.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No +26973.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +26973.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No +27021.jpg Is this artwork created by miniaturist, german? Please answer yes or no. Yes +27021.jpg Is this artwork created by trinquesse, louis-rolland? Please answer yes or no. No +27662.jpg Does this artwork belong to the type of still-life? Please answer yes or no. Yes +27662.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No +27936.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes +27936.jpg Does this artwork belong to the type of interior? Please answer yes or no. No +28039.jpg Is this artwork displayed in cappella palatina, palermo? Please answer yes or no. Yes +28039.jpg Is this artwork displayed in musée des beaux-arts, chambéry? Please answer yes or no. No +28345.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +28345.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No +28400.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +28400.jpg Does this artwork belong to the type of portrait? Please answer yes or no. No +28698.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +28698.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No +28758.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +28758.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No +28974.jpg Is this artwork titled prayer before the meal? Please answer yes or no. Yes +28974.jpg Is this artwork titled rest in the mountains? Please answer yes or no. No +29266.jpg Is this artwork created by palma vecchio? Please answer yes or no. Yes +29266.jpg Is this artwork created by maris, jacobus hendricus? Please answer yes or no. No +30443.jpg Is this artwork titled the crucifixion with sts jerome and christopher? Please answer yes or no. Yes +30443.jpg Is this artwork titled tomb of michelangelo (detail)? Please answer yes or no. No +3085.jpg Is this artwork created by bartsius, willem? Please answer yes or no. Yes +3085.jpg Is this artwork created by oehme, ernst ferdinand? Please answer yes or no. No +30875.jpg Is this artwork created by pomarancio? Please answer yes or no. Yes +30875.jpg Is this artwork created by steen, jan? Please answer yes or no. No +3114.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +3114.jpg Does this artwork belong to the type of study? Please answer yes or no. No +31808.jpg Is this artwork created by raffaello sanzio? Please answer yes or no. Yes +31808.jpg Is this artwork created by simon von taisten? Please answer yes or no. No +32147.jpg Is this artwork titled lucretia? Please answer yes or no. Yes +32147.jpg Is this artwork titled rinaldo abandoning armida (detail)? Please answer yes or no. No +3241.jpg Is this artwork titled holy family? Please answer yes or no. Yes +3241.jpg Is this artwork titled friedrich iii, the wise, and johann i, the constant, electors of saxony? Please answer yes or no. No +33017.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +33017.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No +33069.jpg Does this artwork belong to the type of historical? Please answer yes or no. Yes +33069.jpg Does this artwork belong to the type of interior? Please answer yes or no. No +33173.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +33173.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No +33753.jpg Is this artwork titled vanitas? Please answer yes or no. Yes +33753.jpg Is this artwork titled legend of st francis: 18. apparition at arles (detail)? Please answer yes or no. No +33854.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +33854.jpg Does this artwork belong to the type of study? Please answer yes or no. No +339.jpg Is this artwork displayed in staatliche museen, berlin? Please answer yes or no. Yes +339.jpg Is this artwork displayed in national museum of religious carvings, valladolid? Please answer yes or no. No +33933.jpg Is this artwork titled madonna and child? Please answer yes or no. Yes +33933.jpg Is this artwork titled the bacino di san marco? Please answer yes or no. No +3404.jpg Is this artwork displayed in szépmûvészeti múzeum, budapest? Please answer yes or no. Yes +3404.jpg Is this artwork displayed in s. eustorgio, milan? Please answer yes or no. No +34109.jpg Is this artwork displayed in national gallery of art, washington? Please answer yes or no. Yes +34109.jpg Is this artwork displayed in abbey church of sainte-foy, conques? Please answer yes or no. No +34363.jpg Is this artwork displayed in museo del prado, madrid? Please answer yes or no. Yes +34363.jpg Is this artwork displayed in state tretyakov gallery, moscow? Please answer yes or no. No +34539.jpg Is this artwork titled the victory of eucharistic truth over heresy? Please answer yes or no. Yes +34539.jpg Is this artwork titled a sunday afternoon on the ile de la grande jatte? Please answer yes or no. No +34627.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes +34627.jpg Does this artwork belong to the type of genre? Please answer yes or no. No +34638.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +34638.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No +34669.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes +34669.jpg Does this artwork belong to the type of historical? Please answer yes or no. No +35345.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +35345.jpg Does this artwork belong to the type of landscape? Please answer yes or no. No +35439.jpg Is this artwork titled madonna and child with a host of musical angels? Please answer yes or no. Yes +35439.jpg Is this artwork titled garden in fontenay? Please answer yes or no. No +35460.jpg Is this artwork created by schinkel, karl friedrich? Please answer yes or no. Yes +35460.jpg Is this artwork created by giolfino, bartolomeo? Please answer yes or no. No +35486.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +35486.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No +35513.jpg Is this artwork created by schongauer, martin? Please answer yes or no. Yes +35513.jpg Is this artwork created by cassioli, amos? Please answer yes or no. No +3552.jpg Is this artwork titled madonna degli alberetti? Please answer yes or no. Yes +3552.jpg Is this artwork titled peter gillis? Please answer yes or no. No +35658.jpg Is this artwork created by sebastiano del piombo? Please answer yes or no. Yes +35658.jpg Is this artwork created by jacobsz., dirck? Please answer yes or no. No +35736.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +35736.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No +35861.jpg Does this artwork belong to the type of interior? Please answer yes or no. Yes +35861.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No +36805.jpg Is this artwork titled weir? Please answer yes or no. Yes +36805.jpg Is this artwork titled view of the window wall? Please answer yes or no. No +36966.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes +36966.jpg Does this artwork belong to the type of religious? Please answer yes or no. No +37010.jpg Is this artwork titled madonna and child with the young st john? Please answer yes or no. Yes +37010.jpg Is this artwork titled sketch for attila? Please answer yes or no. No +37077.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +37077.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No +37439.jpg Is this artwork titled the message? Please answer yes or no. Yes +37439.jpg Is this artwork titled the descent from the cross? Please answer yes or no. No +37819.jpg Is this artwork created by tiepolo, giovanni battista? Please answer yes or no. Yes +37819.jpg Is this artwork created by kerricx, willem ignatius? Please answer yes or no. No +37866.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes +37866.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No +381.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +381.jpg Does this artwork exist in the form of architecture? Please answer yes or no. No +38178.jpg Is this artwork created by tintoretto? Please answer yes or no. Yes +38178.jpg Is this artwork created by morel, jean-baptiste? Please answer yes or no. No +38536.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +38536.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No +38546.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +38546.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +38694.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +38694.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +38740.jpg Is this artwork displayed in musée toulouse-lautrec, albi? Please answer yes or no. Yes +38740.jpg Is this artwork displayed in kupferstichkabinett, gotha? Please answer yes or no. No +38881.jpg Does this artwork belong to the type of genre? Please answer yes or no. Yes +38881.jpg Does this artwork belong to the type of religious? Please answer yes or no. No +38993.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +38993.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No +39026.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +39026.jpg Does this artwork belong to the type of historical? Please answer yes or no. No +39124.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +39124.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No +39188.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +39188.jpg Does this artwork exist in the form of architecture? Please answer yes or no. No +39482.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +39482.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No +39556.jpg Is this artwork created by unknown master, dutch? Please answer yes or no. Yes +39556.jpg Is this artwork created by cuyp, benjamin gerritsz.? Please answer yes or no. No +41036.jpg Is this artwork displayed in kunsthistorisches museum, vienna? Please answer yes or no. Yes +41036.jpg Is this artwork displayed in national museum of art, minsk? Please answer yes or no. No +41371.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +41371.jpg Does this artwork exist in the form of architecture? Please answer yes or no. No +41484.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +41484.jpg Does this artwork belong to the type of historical? Please answer yes or no. No +41594.jpg Is this artwork created by veronese, paolo? Please answer yes or no. Yes +41594.jpg Is this artwork created by jeaurat, etienne? Please answer yes or no. No +416.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes +416.jpg Does this artwork exist in the form of others? Please answer yes or no. No +41653.jpg Is this artwork titled view of the sala del collegio? Please answer yes or no. Yes +41653.jpg Is this artwork titled reine lefebvre and margot? Please answer yes or no. No +41944.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +41944.jpg Does this artwork exist in the form of mosaic? Please answer yes or no. No +42152.jpg Is this artwork titled the pieterskerk in leiden? Please answer yes or no. Yes +42152.jpg Is this artwork titled portrait of cardinal reginald pole? Please answer yes or no. No +42288.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +42288.jpg Does this artwork exist in the form of stained-glass? Please answer yes or no. No +42303.jpg Is this artwork displayed in art museum, cincinnati? Please answer yes or no. Yes +42303.jpg Is this artwork displayed in banca del monte di bologna e ravenna, bologna? Please answer yes or no. No +42401.jpg Is this artwork created by waldmüller, fedinand georg? Please answer yes or no. Yes +42401.jpg Is this artwork created by seeman, enoch? Please answer yes or no. No +42447.jpg Is this artwork displayed in musée du louvre, paris? Please answer yes or no. Yes +42447.jpg Is this artwork displayed in santa catarina, pisa? Please answer yes or no. No +42585.jpg Is this artwork created by werff, pieter van der? Please answer yes or no. Yes +42585.jpg Is this artwork created by domenichini, apollonio? Please answer yes or no. No +42706.jpg Is this artwork displayed in musée du louvre, paris? Please answer yes or no. Yes +42706.jpg Is this artwork displayed in galleria nazionale d'arte moderna e contemporanea, rome? Please answer yes or no. No +42796.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +42796.jpg Is this artwork displayed in museo di san salvi, florence? Please answer yes or no. No +42857.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes +42857.jpg Does this artwork belong to the type of study? Please answer yes or no. No +42905.jpg Is this artwork created by wit, jacob de? Please answer yes or no. Yes +42905.jpg Is this artwork created by vittone, bernardo antonio? Please answer yes or no. No +42941.jpg Is this artwork created by witte, emanuel de? Please answer yes or no. Yes +42941.jpg Is this artwork created by bicci di neri? Please answer yes or no. No +42956.jpg Is this artwork titled view of rome with the tiberand castel sant'angelo? Please answer yes or no. Yes +42956.jpg Is this artwork titled st bonaventure enters the franciscan order? Please answer yes or no. No +42987.jpg Is this artwork created by witz, konrad? Please answer yes or no. Yes +42987.jpg Is this artwork created by christus, petrus? Please answer yes or no. No +43142.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes +43142.jpg Does this artwork belong to the type of interior? Please answer yes or no. No +43175.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes +43175.jpg Is this artwork displayed in smith college museum of art, northampton? Please answer yes or no. No +43349.jpg Is this artwork created by zuccarelli, francesco? Please answer yes or no. Yes +43349.jpg Is this artwork created by baccanelli, giovanni antonio di giulio? Please answer yes or no. No +43445.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +43445.jpg Does this artwork belong to the type of interior? Please answer yes or no. No +4836.jpg Is this artwork displayed in villa cornaro, piombino dese? Please answer yes or no. Yes +4836.jpg Is this artwork displayed in palais saint-vaast, arras? Please answer yes or no. No +5227.jpg Is this artwork created by botticelli, sandro? Please answer yes or no. Yes +5227.jpg Is this artwork created by vigri, caterina? Please answer yes or no. No +526.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +526.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No +5906.jpg Is this artwork created by bronzino, agnolo? Please answer yes or no. Yes +5906.jpg Is this artwork created by pellegrino da san daniele? Please answer yes or no. No +6168.jpg Does this artwork exist in the form of graphics? Please answer yes or no. Yes +6168.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No +6297.jpg Is this artwork titled peasants making merry outside a tavern 'the swan'? Please answer yes or no. Yes +6297.jpg Is this artwork titled allegory of quietude? Please answer yes or no. No +6478.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes +6478.jpg Does this artwork belong to the type of genre? Please answer yes or no. No +6969.jpg Is this artwork titled letizia ramolino bonaparte? Please answer yes or no. Yes +6969.jpg Is this artwork titled job and his daughters? Please answer yes or no. No +701.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +701.jpg Does this artwork exist in the form of others? Please answer yes or no. No +7702.jpg Is this artwork titled reine lefebvre and margot? Please answer yes or no. Yes +7702.jpg Is this artwork titled fire in the oil depot at san marcuola? Please answer yes or no. No +8101.jpg Is this artwork displayed in museu de arte, são paulo? Please answer yes or no. Yes +8101.jpg Is this artwork displayed in national széchényi library, budapest? Please answer yes or no. No +815.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +815.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No +8797.jpg Is this artwork created by coecke van aelst, pieter? Please answer yes or no. Yes +8797.jpg Is this artwork created by abaquesne, masséot? Please answer yes or no. No +8885.jpg Is this artwork displayed in art museum, saint louis? Please answer yes or no. Yes +8885.jpg Is this artwork displayed in museo civico d'arte antica, turin? Please answer yes or no. No +9153.jpg Is this artwork displayed in galleria nazionale, parma? Please answer yes or no. Yes +9153.jpg Is this artwork displayed in hospital de san bernardo, seville? Please answer yes or no. No +9395.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +9395.jpg Does this artwork exist in the form of stained-glass? Please answer yes or no. No +9405.jpg Is this artwork created by courbet, gustave? Please answer yes or no. Yes +9405.jpg Is this artwork created by milani, aureliano? Please answer yes or no. No +9599.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes +9599.jpg Does this artwork exist in the form of ceramics? Please answer yes or no. No +995.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes +995.jpg Does this artwork exist in the form of painting? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/celebrity.txt b/internvl_chat_audio/eval/mme/Your_Results/celebrity.txt new file mode 100644 index 00000000..e671c331 --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/celebrity.txt @@ -0,0 +1,340 @@ +tt0032138_shot_0395_img_0.jpg Is the actor inside the red bounding box named Frank Morgan? Please answer yes or no. Yes +tt0032138_shot_0395_img_0.jpg Is the actor inside the red bounding box named Eric Schniewind? Please answer yes or no. No +tt0035423_shot_0464_img_0.jpg Is the actor inside the red bounding box called Hugh Jackman? Please answer yes or no. Yes +tt0035423_shot_0464_img_0.jpg Is the actor inside the red bounding box called Lizzie Hopley? Please answer yes or no. No +tt0038650_shot_0737_img_1.jpg Is the person inside the red bounding box called James Stewart? Please answer yes or no. Yes +tt0038650_shot_0737_img_1.jpg Is the person inside the red bounding box called Phil Selway? Please answer yes or no. No +tt0047396_shot_0333_img_0.jpg Is the actor inside the red bounding box named James Stewart? Please answer yes or no. Yes +tt0047396_shot_0333_img_0.jpg Is the actor inside the red bounding box named Ron Blair? Please answer yes or no. No +tt0048545_shot_0124_img_0.jpg Is the actor inside the red bounding box called Natalie Wood? Please answer yes or no. Yes +tt0048545_shot_0124_img_0.jpg Is the actor inside the red bounding box called Rebecca Jackson Mendoza? Please answer yes or no. No +tt0049470_shot_0279_img_0.jpg Is the person inside the red bounding box called James Stewart? Please answer yes or no. Yes +tt0049470_shot_0279_img_0.jpg Is the person inside the red bounding box called Matt Pashkow? Please answer yes or no. No +tt0049730_shot_0273_img_0.jpg Is the person inside the red bounding box called Vera Miles? Please answer yes or no. Yes +tt0049730_shot_0273_img_0.jpg Is the person inside the red bounding box called Addie Yungmee? Please answer yes or no. No +tt0052357_shot_0511_img_0.jpg Is the actor inside the red bounding box called Kim Novak? Please answer yes or no. Yes +tt0052357_shot_0511_img_0.jpg Is the actor inside the red bounding box called Abigail Van Alyn? Please answer yes or no. No +tt0053221_shot_0197_img_0.jpg Is the actor inside the red bounding box named John Wayne? Please answer yes or no. Yes +tt0053221_shot_0197_img_0.jpg Is the actor inside the red bounding box named Claude-Oliver Rudolph? Please answer yes or no. No +tt0054167_shot_0122_img_0.jpg Is the person inside the red bounding box called Anna Massey? Please answer yes or no. Yes +tt0054167_shot_0122_img_0.jpg Is the person inside the red bounding box called Eddie Tagoe? Please answer yes or no. No +tt0056869_shot_0320_img_0.jpg Is the person inside the red bounding box called Tippi Hedren? Please answer yes or no. Yes +tt0056869_shot_0320_img_0.jpg Is the person inside the red bounding box called Denise Mack? Please answer yes or no. No +tt0056923_shot_0835_img_0.jpg Is the actor inside the red bounding box called Audrey Hepburn? Please answer yes or no. Yes +tt0056923_shot_0835_img_0.jpg Is the actor inside the red bounding box called Chris April? Please answer yes or no. No +tt0057115_shot_0686_img_0.jpg Is the person inside the red bounding box named James Garner? Please answer yes or no. Yes +tt0057115_shot_0686_img_0.jpg Is the person inside the red bounding box named Chutimon Chuengcharoensukying? Please answer yes or no. No +tt0058331_shot_0353_img_0.jpg Is the actor inside the red bounding box named Julie Andrews? Please answer yes or no. Yes +tt0058331_shot_0353_img_0.jpg Is the actor inside the red bounding box named Ed Geldart? Please answer yes or no. No +tt0058461_shot_0901_img_0.jpg Is the actor inside the red bounding box called Gian Maria Volontè? Please answer yes or no. Yes +tt0058461_shot_0901_img_0.jpg Is the actor inside the red bounding box called Jennifer Connelly? Please answer yes or no. No +tt0061418_shot_0148_img_0.jpg Is the actor inside the red bounding box named Faye Dunaway? Please answer yes or no. Yes +tt0061418_shot_0148_img_0.jpg Is the actor inside the red bounding box named Warona Seane? Please answer yes or no. No +tt0061722_shot_0259_img_0.jpg Is the actor inside the red bounding box called Dustin Hoffman? Please answer yes or no. Yes +tt0061722_shot_0259_img_0.jpg Is the actor inside the red bounding box called Christopher Olsen? Please answer yes or no. No +tt0062622_shot_0291_img_0.jpg Is the actor inside the red bounding box named Keir Dullea? Please answer yes or no. Yes +tt0062622_shot_0291_img_0.jpg Is the actor inside the red bounding box named Frank Albanese? Please answer yes or no. No +tt0063442_shot_0702_img_0.jpg Is the actor inside the red bounding box called Linda Harrison? Please answer yes or no. Yes +tt0063442_shot_0702_img_0.jpg Is the actor inside the red bounding box called Michael McKean? Please answer yes or no. No +tt0064115_shot_0367_img_0.jpg Is the actor inside the red bounding box named Robert Redford? Please answer yes or no. Yes +tt0064115_shot_0367_img_0.jpg Is the actor inside the red bounding box named Cooper Murray? Please answer yes or no. No +tt0064665_shot_0300_img_0.jpg Is the actor inside the red bounding box called Jon Voight? Please answer yes or no. Yes +tt0064665_shot_0300_img_0.jpg Is the actor inside the red bounding box called Harvey Meyer? Please answer yes or no. No +tt0065214_shot_0366_img_0.jpg Is the person inside the red bounding box called Robert Ryan? Please answer yes or no. Yes +tt0065214_shot_0366_img_0.jpg Is the person inside the red bounding box called Victor Verhaeghe? Please answer yes or no. No +tt0065724_shot_0320_img_1.jpg Is the person inside the red bounding box named Karen Black? Please answer yes or no. Yes +tt0065724_shot_0320_img_1.jpg Is the person inside the red bounding box named Nick Discenza? Please answer yes or no. No +tt0066026_shot_0085_img_0.jpg Is the person inside the red bounding box called Donald Sutherland? Please answer yes or no. Yes +tt0066026_shot_0085_img_0.jpg Is the person inside the red bounding box called Michael Wollet? Please answer yes or no. No +tt0066921_shot_0631_img_0.jpg Is the actor inside the red bounding box called Malcolm McDowell? Please answer yes or no. Yes +tt0066921_shot_0631_img_0.jpg Is the actor inside the red bounding box called Darling Légitimus? Please answer yes or no. No +tt0067116_shot_0122_img_0.jpg Is the actor inside the red bounding box called Gene Hackman? Please answer yes or no. Yes +tt0067116_shot_0122_img_0.jpg Is the actor inside the red bounding box called Russell G. Jones? Please answer yes or no. No +tt0068646_shot_0166_img_0.jpg Is the actor inside the red bounding box called Marlon Brando? Please answer yes or no. Yes +tt0068646_shot_0166_img_0.jpg Is the actor inside the red bounding box called Voltaire Sterling? Please answer yes or no. No +tt0069762_shot_0723_img_0.jpg Is the person inside the red bounding box named Sissy Spacek? Please answer yes or no. Yes +tt0069762_shot_0723_img_0.jpg Is the person inside the red bounding box named Monica Giordano? Please answer yes or no. No +tt0070047_shot_0255_img_0.jpg Is the actor inside the red bounding box called Ellen Burstyn? Please answer yes or no. Yes +tt0070047_shot_0255_img_0.jpg Is the actor inside the red bounding box called Shawnee Smith? Please answer yes or no. No +tt0070379_shot_0569_img_0.jpg Is the actor inside the red bounding box named Richard Romanus? Please answer yes or no. Yes +tt0070379_shot_0569_img_0.jpg Is the actor inside the red bounding box named Valerie Colgan? Please answer yes or no. No +tt0070511_shot_0639_img_0.jpg Is the person inside the red bounding box called Dustin Hoffman? Please answer yes or no. Yes +tt0070511_shot_0639_img_0.jpg Is the person inside the red bounding box called Fernando Lueches? Please answer yes or no. No +tt0070735_shot_0818_img_0.jpg Is the person inside the red bounding box named Robert Redford? Please answer yes or no. Yes +tt0070735_shot_0818_img_0.jpg Is the person inside the red bounding box named Ellin Dennis? Please answer yes or no. No +tt0070849_shot_0021_img_1.jpg Is the person inside the red bounding box named Maria Schneider? Please answer yes or no. Yes +tt0070849_shot_0021_img_1.jpg Is the person inside the red bounding box named Mary Kellogg? Please answer yes or no. No +tt0071315_shot_0153_img_0.jpg Is the actor inside the red bounding box named Faye Dunaway? Please answer yes or no. Yes +tt0071315_shot_0153_img_0.jpg Is the actor inside the red bounding box named Kelly Hitman? Please answer yes or no. No +tt0071562_shot_0684_img_0.jpg Is the actor inside the red bounding box named Al Pacino? Please answer yes or no. Yes +tt0071562_shot_0684_img_0.jpg Is the actor inside the red bounding box named Debie Jarczewski? Please answer yes or no. No +tt0072684_shot_0512_img_1.jpg Is the person inside the red bounding box named Marisa Berenson? Please answer yes or no. Yes +tt0072684_shot_0512_img_1.jpg Is the person inside the red bounding box named Graham Bohea? Please answer yes or no. No +tt0073195_shot_0280_img_0.jpg Is the actor inside the red bounding box named Roy Scheider? Please answer yes or no. Yes +tt0073195_shot_0280_img_0.jpg Is the actor inside the red bounding box named Abdul Qadir Farookh? Please answer yes or no. No +tt0073629_shot_0700_img_0.jpg Is the person inside the red bounding box named Barry Bostwick? Please answer yes or no. Yes +tt0073629_shot_0700_img_0.jpg Is the person inside the red bounding box named Johnny Galecki? Please answer yes or no. No +tt0074119_shot_0814_img_0.jpg Is the actor inside the red bounding box called Robert Redford? Please answer yes or no. Yes +tt0074119_shot_0814_img_0.jpg Is the actor inside the red bounding box called Delroy Lindo? Please answer yes or no. No +tt0074285_shot_0535_img_1.jpg Is the person inside the red bounding box named William Katt? Please answer yes or no. Yes +tt0074285_shot_0535_img_1.jpg Is the person inside the red bounding box named Stephen Rider? Please answer yes or no. No +tt0075148_shot_0618_img_0.jpg Is the actor inside the red bounding box called Sylvester Stallone? Please answer yes or no. Yes +tt0075148_shot_0618_img_0.jpg Is the actor inside the red bounding box called Eric Hatch? Please answer yes or no. No +tt0075686_shot_0373_img_0.jpg Is the actor inside the red bounding box called Woody Allen? Please answer yes or no. Yes +tt0075686_shot_0373_img_0.jpg Is the actor inside the red bounding box called Penny Wallace? Please answer yes or no. No +tt0076729_shot_0451_img_0.jpg Is the actor inside the red bounding box called Sally Field? Please answer yes or no. Yes +tt0076729_shot_0451_img_0.jpg Is the actor inside the red bounding box called Giorgio Libassi? Please answer yes or no. No +tt0076759_shot_0930_img_0.jpg Is the actor inside the red bounding box called Harrison Ford? Please answer yes or no. Yes +tt0076759_shot_0930_img_0.jpg Is the actor inside the red bounding box called Ryoko Sadoshima? Please answer yes or no. No +tt0077402_shot_1220_img_0.jpg Is the person inside the red bounding box named Scott H. Reiniger? Please answer yes or no. Yes +tt0077402_shot_1220_img_0.jpg Is the person inside the red bounding box named Chris Delaney? Please answer yes or no. No +tt0077405_shot_0150_img_0.jpg Is the actor inside the red bounding box named Sam Shepard? Please answer yes or no. Yes +tt0077405_shot_0150_img_0.jpg Is the actor inside the red bounding box named Bijou Phillips? Please answer yes or no. No +tt0077416_shot_1442_img_0.jpg Is the person inside the red bounding box named Robert De Niro? Please answer yes or no. Yes +tt0077416_shot_1442_img_0.jpg Is the person inside the red bounding box named Stu Smith? Please answer yes or no. No +tt0077651_shot_0133_img_0.jpg Is the person inside the red bounding box called Jamie Lee Curtis? Please answer yes or no. Yes +tt0077651_shot_0133_img_0.jpg Is the person inside the red bounding box called Paris Arrowsmith? Please answer yes or no. No +tt0078788_shot_1434_img_0.jpg Is the person inside the red bounding box called Martin Sheen? Please answer yes or no. Yes +tt0078788_shot_1434_img_0.jpg Is the person inside the red bounding box called Le Capriccio Français? Please answer yes or no. No +tt0078841_shot_0692_img_0.jpg Is the actor inside the red bounding box named Shirley MacLaine? Please answer yes or no. Yes +tt0078841_shot_0692_img_0.jpg Is the actor inside the red bounding box named Tomas Choy? Please answer yes or no. No +tt0079417_shot_0735_img_0.jpg Is the actor inside the red bounding box called Meryl Streep? Please answer yes or no. Yes +tt0079417_shot_0735_img_0.jpg Is the actor inside the red bounding box called Ross Lacy? Please answer yes or no. No +tt0079470_shot_0798_img_0.jpg Is the person inside the red bounding box named Eric Idle? Please answer yes or no. Yes +tt0079470_shot_0798_img_0.jpg Is the person inside the red bounding box named Quincy Taylor? Please answer yes or no. No +tt0079945_shot_1411_img_0.jpg Is the person inside the red bounding box named Persis Khambatta? Please answer yes or no. Yes +tt0079945_shot_1411_img_0.jpg Is the person inside the red bounding box named Alison Waddell? Please answer yes or no. No +tt0080339_shot_0711_img_0.jpg Is the actor inside the red bounding box named Robert Hays? Please answer yes or no. Yes +tt0080339_shot_0711_img_0.jpg Is the actor inside the red bounding box named Grace Sullivan? Please answer yes or no. No +tt0080684_shot_1574_img_2.jpg Is the actor inside the red bounding box called Mark Hamill? Please answer yes or no. Yes +tt0080684_shot_1574_img_2.jpg Is the actor inside the red bounding box called Rodion Salnikov? Please answer yes or no. No +tt0081505_shot_0449_img_0.jpg Is the actor inside the red bounding box called Shelley Duvall? Please answer yes or no. Yes +tt0081505_shot_0449_img_0.jpg Is the actor inside the red bounding box called Antony Carrick? Please answer yes or no. No +tt0082089_shot_0046_img_0.jpg Is the actor inside the red bounding box named Kathleen Turner? Please answer yes or no. Yes +tt0082089_shot_0046_img_0.jpg Is the actor inside the red bounding box named Aaron Henderson? Please answer yes or no. No +tt0082198_shot_1353_img_0.jpg Is the person inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes +tt0082198_shot_1353_img_0.jpg Is the person inside the red bounding box named Tim Herlihy? Please answer yes or no. No +tt0082971_shot_0831_img_0.jpg Is the actor inside the red bounding box called Harrison Ford? Please answer yes or no. Yes +tt0082971_shot_0831_img_0.jpg Is the actor inside the red bounding box called Richard Angarola? Please answer yes or no. No +tt0083658_shot_0963_img_0.jpg Is the person inside the red bounding box called Rutger Hauer? Please answer yes or no. Yes +tt0083658_shot_0963_img_0.jpg Is the person inside the red bounding box called Stéphane Julien? Please answer yes or no. No +tt0083866_shot_0364_img_0.jpg Is the actor inside the red bounding box called Robert MacNaughton? Please answer yes or no. Yes +tt0083866_shot_0364_img_0.jpg Is the actor inside the red bounding box called Seam Turay? Please answer yes or no. No +tt0083907_shot_0633_img_0.jpg Is the actor inside the red bounding box named Bruce Campbell? Please answer yes or no. Yes +tt0083907_shot_0633_img_0.jpg Is the actor inside the red bounding box named Kaden Leos? Please answer yes or no. No +tt0083929_shot_0405_img_0.jpg Is the actor inside the red bounding box named Jennifer Jason Leigh? Please answer yes or no. Yes +tt0083929_shot_0405_img_0.jpg Is the actor inside the red bounding box named Eric D. Sandgren? Please answer yes or no. No +tt0084726_shot_0283_img_0.jpg Is the actor inside the red bounding box named Leonard Nimoy? Please answer yes or no. Yes +tt0084726_shot_0283_img_0.jpg Is the actor inside the red bounding box named John Cusack? Please answer yes or no. No +tt0086190_shot_0815_img_0.jpg Is the actor inside the red bounding box named Carrie Fisher? Please answer yes or no. Yes +tt0086190_shot_0815_img_0.jpg Is the actor inside the red bounding box named Ernie Adams? Please answer yes or no. No +tt0086250_shot_1079_img_0.jpg Is the actor inside the red bounding box called Steven Bauer? Please answer yes or no. Yes +tt0086250_shot_1079_img_0.jpg Is the actor inside the red bounding box called Bill Nunn? Please answer yes or no. No +tt0086856_shot_0929_img_0.jpg Is the actor inside the red bounding box called Peter Weller? Please answer yes or no. Yes +tt0086856_shot_0929_img_0.jpg Is the actor inside the red bounding box called Tracee Cocco? Please answer yes or no. No +tt0086879_shot_0158_img_0.jpg Is the person inside the red bounding box called Elizabeth Berridge? Please answer yes or no. Yes +tt0086879_shot_0158_img_0.jpg Is the person inside the red bounding box called Ralph Ineson? Please answer yes or no. No +tt0087332_shot_0798_img_0.jpg Is the person inside the red bounding box called Bill Murray? Please answer yes or no. Yes +tt0087332_shot_0798_img_0.jpg Is the person inside the red bounding box called Jiao Xu? Please answer yes or no. No +tt0087469_shot_0049_img_2.jpg Is the person inside the red bounding box named Harrison Ford? Please answer yes or no. Yes +tt0087469_shot_0049_img_2.jpg Is the person inside the red bounding box named Paulo Benedeti? Please answer yes or no. No +tt0088847_shot_0109_img_0.jpg Is the actor inside the red bounding box named Anthony Michael Hall? Please answer yes or no. Yes +tt0088847_shot_0109_img_0.jpg Is the actor inside the red bounding box named Luis Javier? Please answer yes or no. No +tt0088944_shot_0634_img_0.jpg Is the actor inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes +tt0088944_shot_0634_img_0.jpg Is the actor inside the red bounding box named Shaine Jones? Please answer yes or no. No +tt0088993_shot_0569_img_0.jpg Is the actor inside the red bounding box called George A. Romero? Please answer yes or no. Yes +tt0088993_shot_0569_img_0.jpg Is the actor inside the red bounding box called James Eckhouse? Please answer yes or no. No +tt0089218_shot_0327_img_0.jpg Is the person inside the red bounding box named Sean Astin? Please answer yes or no. Yes +tt0089218_shot_0327_img_0.jpg Is the person inside the red bounding box named Dan Hunter? Please answer yes or no. No +tt0089881_shot_0034_img_0.jpg Is the actor inside the red bounding box called Tatsuya Nakadai? Please answer yes or no. Yes +tt0089881_shot_0034_img_0.jpg Is the actor inside the red bounding box called Nancy Vee? Please answer yes or no. No +tt0090022_shot_0464_img_0.jpg Is the actor inside the red bounding box called Scott Glenn? Please answer yes or no. Yes +tt0090022_shot_0464_img_0.jpg Is the actor inside the red bounding box called Robert Ryan? Please answer yes or no. No +tt0090605_shot_0344_img_0.jpg Is the person inside the red bounding box called Sigourney Weaver? Please answer yes or no. Yes +tt0090605_shot_0344_img_0.jpg Is the person inside the red bounding box called Lia Beldam? Please answer yes or no. No +tt0090756_shot_0135_img_0.jpg Is the person inside the red bounding box named Laura Dern? Please answer yes or no. Yes +tt0090756_shot_0135_img_0.jpg Is the person inside the red bounding box named Keith Frost? Please answer yes or no. No +tt0091042_shot_0098_img_0.jpg Is the person inside the red bounding box called Matthew Broderick? Please answer yes or no. Yes +tt0091042_shot_0098_img_0.jpg Is the person inside the red bounding box called Mina E. Mina? Please answer yes or no. No +tt0091738_shot_0073_img_1.jpg Is the actor inside the red bounding box called Kathleen Turner? Please answer yes or no. Yes +tt0091738_shot_0073_img_1.jpg Is the actor inside the red bounding box called Pat Kiernan? Please answer yes or no. No +tt0091867_shot_0422_img_2.jpg Is the person inside the red bounding box named Simon Callow? Please answer yes or no. Yes +tt0091867_shot_0422_img_2.jpg Is the person inside the red bounding box named Rusty Goffe? Please answer yes or no. No +tt0092099_shot_0455_img_1.jpg Is the person inside the red bounding box called Tom Cruise? Please answer yes or no. Yes +tt0092099_shot_0455_img_1.jpg Is the person inside the red bounding box called Carol Krolick? Please answer yes or no. No +tt0092699_shot_0208_img_0.jpg Is the actor inside the red bounding box called William Hurt? Please answer yes or no. Yes +tt0092699_shot_0208_img_0.jpg Is the actor inside the red bounding box called Hildur Ruriks? Please answer yes or no. No +tt0093565_shot_0409_img_0.jpg Is the actor inside the red bounding box named Cher? Please answer yes or no. Yes +tt0093565_shot_0409_img_0.jpg Is the actor inside the red bounding box named Mark Brady? Please answer yes or no. No +tt0093748_shot_0346_img_0.jpg Is the actor inside the red bounding box called John Candy? Please answer yes or no. Yes +tt0093748_shot_0346_img_0.jpg Is the actor inside the red bounding box called Sarah Heller? Please answer yes or no. No +tt0093773_shot_0212_img_0.jpg Is the person inside the red bounding box named Jesse Ventura? Please answer yes or no. Yes +tt0093773_shot_0212_img_0.jpg Is the person inside the red bounding box named Akio Mitamura? Please answer yes or no. No +tt0093779_shot_1047_img_0.jpg Is the person inside the red bounding box named Peter Falk? Please answer yes or no. Yes +tt0093779_shot_1047_img_0.jpg Is the person inside the red bounding box named Lisa Ann Walter? Please answer yes or no. No +tt0094226_shot_0237_img_2.jpg Is the actor inside the red bounding box called Kevin Costner? Please answer yes or no. Yes +tt0094226_shot_0237_img_2.jpg Is the actor inside the red bounding box called Colin Hill? Please answer yes or no. No +tt0094737_shot_0567_img_0.jpg Is the person inside the red bounding box called Tom Hanks? Please answer yes or no. Yes +tt0094737_shot_0567_img_0.jpg Is the person inside the red bounding box called Chris McHallem? Please answer yes or no. No +tt0095016_shot_1170_img_0.jpg Is the actor inside the red bounding box called Paul Gleason? Please answer yes or no. Yes +tt0095016_shot_1170_img_0.jpg Is the actor inside the red bounding box called Carl Palmer? Please answer yes or no. No +tt0095250_shot_0509_img_0.jpg Is the actor inside the red bounding box named Jean Reno? Please answer yes or no. Yes +tt0095250_shot_0509_img_0.jpg Is the actor inside the red bounding box named Ralph Meyering Jr.? Please answer yes or no. No +tt0095765_shot_0008_img_0.jpg Is the actor inside the red bounding box called Antonella Attili? Please answer yes or no. Yes +tt0095765_shot_0008_img_0.jpg Is the actor inside the red bounding box called Amber Estrada? Please answer yes or no. No +tt0095953_shot_0412_img_0.jpg Is the person inside the red bounding box named Tom Cruise? Please answer yes or no. Yes +tt0095953_shot_0412_img_0.jpg Is the person inside the red bounding box named Lara Mulcahy? Please answer yes or no. No +tt0096320_shot_0085_img_0.jpg Is the actor inside the red bounding box called Arnold Schwarzenegger? Please answer yes or no. Yes +tt0096320_shot_0085_img_0.jpg Is the actor inside the red bounding box called Dan Duran? Please answer yes or no. No +tt0096754_shot_0570_img_1.jpg Is the person inside the red bounding box named Todd Graff? Please answer yes or no. Yes +tt0096754_shot_0570_img_1.jpg Is the person inside the red bounding box named Guy Carleton? Please answer yes or no. No +tt0096874_shot_0647_img_0.jpg Is the actor inside the red bounding box named Michael J. Fox? Please answer yes or no. Yes +tt0096874_shot_0647_img_0.jpg Is the actor inside the red bounding box named Momoko Komatsu? Please answer yes or no. No +tt0096895_shot_0819_img_1.jpg Is the person inside the red bounding box called Michael Keaton? Please answer yes or no. Yes +tt0096895_shot_0819_img_1.jpg Is the person inside the red bounding box called Ben Foster? Please answer yes or no. No +tt0097216_shot_0381_img_0.jpg Is the actor inside the red bounding box named Danny Aiello? Please answer yes or no. Yes +tt0097216_shot_0381_img_0.jpg Is the actor inside the red bounding box named Taissa Farmiga? Please answer yes or no. No +tt0097428_shot_0106_img_0.jpg Is the actor inside the red bounding box named Bill Murray? Please answer yes or no. Yes +tt0097428_shot_0106_img_0.jpg Is the actor inside the red bounding box named Michael Fawcett? Please answer yes or no. No +tt0097576_shot_1010_img_2.jpg Is the actor inside the red bounding box named Harrison Ford? Please answer yes or no. Yes +tt0097576_shot_1010_img_2.jpg Is the actor inside the red bounding box named M. Emmet Walsh? Please answer yes or no. No +tt0098635_shot_0556_img_0.jpg Is the actor inside the red bounding box named Meg Ryan? Please answer yes or no. Yes +tt0098635_shot_0556_img_0.jpg Is the actor inside the red bounding box named Tom Branch? Please answer yes or no. No +tt0098724_shot_0474_img_0.jpg Is the person inside the red bounding box named Andie MacDowell? Please answer yes or no. Yes +tt0098724_shot_0474_img_0.jpg Is the person inside the red bounding box named Linda Taylor? Please answer yes or no. No +tt0099423_shot_1010_img_0.jpg Is the person inside the red bounding box called Bruce Willis? Please answer yes or no. Yes +tt0099423_shot_1010_img_0.jpg Is the person inside the red bounding box called Trevor Eve? Please answer yes or no. No +tt0099487_shot_0123_img_0.jpg Is the actor inside the red bounding box named Johnny Depp? Please answer yes or no. Yes +tt0099487_shot_0123_img_0.jpg Is the actor inside the red bounding box named Farrah Forke? Please answer yes or no. No +tt0099674_shot_1356_img_0.jpg Is the person inside the red bounding box named Al Pacino? Please answer yes or no. Yes +tt0099674_shot_1356_img_0.jpg Is the person inside the red bounding box named Nick Porrazzo? Please answer yes or no. No +tt0099685_shot_1132_img_0.jpg Is the actor inside the red bounding box called Ray Liotta? Please answer yes or no. Yes +tt0099685_shot_1132_img_0.jpg Is the actor inside the red bounding box called Chick Allan? Please answer yes or no. No +tt0099810_shot_0285_img_0.jpg Is the person inside the red bounding box called Alec Baldwin? Please answer yes or no. Yes +tt0099810_shot_0285_img_0.jpg Is the person inside the red bounding box called Jennifer Anglin? Please answer yes or no. No +tt0100157_shot_0365_img_0.jpg Is the actor inside the red bounding box named James Caan? Please answer yes or no. Yes +tt0100157_shot_0365_img_0.jpg Is the actor inside the red bounding box named Bryan Johnson? Please answer yes or no. No +tt0100403_shot_0517_img_0.jpg Is the person inside the red bounding box called Gary Busey? Please answer yes or no. Yes +tt0100403_shot_0517_img_0.jpg Is the person inside the red bounding box called Alfred Tiaki Hotu? Please answer yes or no. No +tt0100405_shot_0786_img_0.jpg Is the actor inside the red bounding box named Jason Alexander? Please answer yes or no. Yes +tt0100405_shot_0786_img_0.jpg Is the actor inside the red bounding box named Alexandra Bastedo? Please answer yes or no. No +tt0101410_shot_0105_img_0.jpg Is the person inside the red bounding box named John Turturro? Please answer yes or no. Yes +tt0101410_shot_0105_img_0.jpg Is the person inside the red bounding box named David Gore? Please answer yes or no. No +tt0102492_shot_0086_img_0.jpg Is the actor inside the red bounding box called Jamie Lee Curtis? Please answer yes or no. Yes +tt0102492_shot_0086_img_0.jpg Is the actor inside the red bounding box called Heidi Fischer? Please answer yes or no. No +tt0103064_shot_1206_img_0.jpg Is the actor inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes +tt0103064_shot_1206_img_0.jpg Is the actor inside the red bounding box named Gigi Lee? Please answer yes or no. No +tt0103064_shot_2602_img_1.jpg Is the person inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes +tt0103064_shot_2602_img_1.jpg Is the person inside the red bounding box named Candice Azzara? Please answer yes or no. No +tt0103776_shot_0719_img_0.jpg Is the person inside the red bounding box called Michael Keaton? Please answer yes or no. Yes +tt0103776_shot_0719_img_0.jpg Is the person inside the red bounding box called Nicholas Rice? Please answer yes or no. No +tt0104036_shot_0336_img_1.jpg Is the person inside the red bounding box named Stephen Rea? Please answer yes or no. Yes +tt0104036_shot_0336_img_1.jpg Is the person inside the red bounding box named Mimi Lizio? Please answer yes or no. No +tt0104257_shot_0477_img_0.jpg Is the person inside the red bounding box named Jack Nicholson? Please answer yes or no. Yes +tt0104257_shot_0477_img_0.jpg Is the person inside the red bounding box named Emma Julia Jacobs? Please answer yes or no. No +tt0104348_shot_0340_img_0.jpg Is the person inside the red bounding box called Ed Harris? Please answer yes or no. Yes +tt0104348_shot_0340_img_0.jpg Is the person inside the red bounding box called Carla Lizzette Mejia? Please answer yes or no. No +tt0105236_shot_0193_img_0.jpg Is the actor inside the red bounding box named Harvey Keitel? Please answer yes or no. Yes +tt0105236_shot_0193_img_0.jpg Is the actor inside the red bounding box named Terence Yin? Please answer yes or no. No +tt0105665_shot_0351_img_0.jpg Is the actor inside the red bounding box named Kyle MacLachlan? Please answer yes or no. Yes +tt0105665_shot_0351_img_0.jpg Is the actor inside the red bounding box named Julia Hsu? Please answer yes or no. No +tt0105695_shot_1436_img_1.jpg Is the person inside the red bounding box called Jaimz Woolvett? Please answer yes or no. Yes +tt0105695_shot_1436_img_1.jpg Is the person inside the red bounding box called Hermione Baddeley? Please answer yes or no. No +tt0106977_shot_1604_img_0.jpg Is the person inside the red bounding box named Tommy Lee Jones? Please answer yes or no. Yes +tt0106977_shot_1604_img_0.jpg Is the person inside the red bounding box named Honey Chhaya? Please answer yes or no. No +tt0107614_shot_0116_img_0.jpg Is the person inside the red bounding box called Sally Field? Please answer yes or no. Yes +tt0107614_shot_0116_img_0.jpg Is the person inside the red bounding box called Arthur Senzy? Please answer yes or no. No +tt0108399_shot_0778_img_0.jpg Is the actor inside the red bounding box called Christopher Walken? Please answer yes or no. Yes +tt0108399_shot_0778_img_0.jpg Is the actor inside the red bounding box called Fiona Sit? Please answer yes or no. No +tt0109831_shot_0298_img_0.jpg Is the person inside the red bounding box called Hugh Grant? Please answer yes or no. Yes +tt0109831_shot_0298_img_0.jpg Is the person inside the red bounding box called Renée Zellweger? Please answer yes or no. No +tt0111280_shot_0258_img_0.jpg Is the actor inside the red bounding box named Gates McFadden? Please answer yes or no. Yes +tt0111280_shot_0258_img_0.jpg Is the actor inside the red bounding box named Michael Angarano? Please answer yes or no. No +tt0111280_shot_1479_img_2.jpg Is the actor inside the red bounding box called William Shatner? Please answer yes or no. Yes +tt0111280_shot_1479_img_2.jpg Is the actor inside the red bounding box called Richard Rohrbough? Please answer yes or no. No +tt0112384_shot_0878_img_0.jpg Is the person inside the red bounding box called Kathleen Quinlan? Please answer yes or no. Yes +tt0112384_shot_0878_img_0.jpg Is the person inside the red bounding box called Veronica Diaz Carranza? Please answer yes or no. No +tt0112641_shot_0412_img_1.jpg Is the actor inside the red bounding box called Robert De Niro? Please answer yes or no. Yes +tt0112641_shot_0412_img_1.jpg Is the actor inside the red bounding box called Pierre Malherbe? Please answer yes or no. No +tt0112740_shot_1056_img_0.jpg Is the person inside the red bounding box named Denzel Washington? Please answer yes or no. Yes +tt0112740_shot_1056_img_0.jpg Is the person inside the red bounding box named Bill Pullman? Please answer yes or no. No +tt0113101_shot_0547_img_0.jpg Is the person inside the red bounding box named Tim Roth? Please answer yes or no. Yes +tt0113101_shot_0547_img_0.jpg Is the person inside the red bounding box named Honey Chhaya? Please answer yes or no. No +tt0114369_shot_1138_img_0.jpg Is the person inside the red bounding box named Brad Pitt? Please answer yes or no. Yes +tt0114369_shot_1138_img_0.jpg Is the person inside the red bounding box named Benjamin Nitze? Please answer yes or no. No +tt0114388_shot_0162_img_0.jpg Is the actor inside the red bounding box called Emma Thompson? Please answer yes or no. Yes +tt0114388_shot_0162_img_0.jpg Is the actor inside the red bounding box called Francis P. Hughes? Please answer yes or no. No +tt0114388_shot_1207_img_1.jpg Is the person inside the red bounding box called Hugh Grant? Please answer yes or no. Yes +tt0114388_shot_1207_img_1.jpg Is the person inside the red bounding box called Zach Hopkins? Please answer yes or no. No +tt0115798_shot_0844_img_1.jpg Is the person inside the red bounding box named Jim Carrey? Please answer yes or no. Yes +tt0115798_shot_0844_img_1.jpg Is the person inside the red bounding box named Renee Herlocker? Please answer yes or no. No +tt0116367_shot_0755_img_0.jpg Is the actor inside the red bounding box named George Clooney? Please answer yes or no. Yes +tt0116367_shot_0755_img_0.jpg Is the actor inside the red bounding box named Ben Crowley? Please answer yes or no. No +tt0116629_shot_1570_img_2.jpg Is the person inside the red bounding box called Will Smith? Please answer yes or no. Yes +tt0116629_shot_1570_img_2.jpg Is the person inside the red bounding box called E. Katherine Kerr? Please answer yes or no. No +tt0116695_shot_0343_img_0.jpg Is the person inside the red bounding box named Tom Cruise? Please answer yes or no. Yes +tt0116695_shot_0343_img_0.jpg Is the person inside the red bounding box named Billy Dee? Please answer yes or no. No +tt0117060_shot_0412_img_0.jpg Is the actor inside the red bounding box called Tom Cruise? Please answer yes or no. Yes +tt0117060_shot_0412_img_0.jpg Is the actor inside the red bounding box called Carrie Lazar? Please answer yes or no. No +tt0117060_shot_1401_img_0.jpg Is the actor inside the red bounding box called Jean Reno? Please answer yes or no. Yes +tt0117060_shot_1401_img_0.jpg Is the actor inside the red bounding box called Jill Teed? Please answer yes or no. No +tt0117381_shot_0798_img_1.jpg Is the person inside the red bounding box called Edward Norton? Please answer yes or no. Yes +tt0117381_shot_0798_img_1.jpg Is the person inside the red bounding box called Michael Tezla? Please answer yes or no. No +tt0117500_shot_2467_img_0.jpg Is the actor inside the red bounding box called Ed Harris? Please answer yes or no. Yes +tt0117500_shot_2467_img_0.jpg Is the actor inside the red bounding box called Paul J.Q. Lee? Please answer yes or no. No +tt0117509_shot_0041_img_0.jpg Is the actor inside the red bounding box named Paul Rudd? Please answer yes or no. Yes +tt0117509_shot_0041_img_0.jpg Is the actor inside the red bounding box named Max Martini? Please answer yes or no. No +tt0117571_shot_0475_img_0.jpg Is the person inside the red bounding box named Neve Campbell? Please answer yes or no. Yes +tt0117571_shot_0475_img_0.jpg Is the person inside the red bounding box named Frank Hoyt Taylor? Please answer yes or no. No +tt0117731_shot_0300_img_0.jpg Is the actor inside the red bounding box called Patrick Stewart? Please answer yes or no. Yes +tt0117731_shot_0300_img_0.jpg Is the actor inside the red bounding box called Debra Montague? Please answer yes or no. No +tt0117731_shot_1067_img_0.jpg Is the actor inside the red bounding box called Patrick Stewart? Please answer yes or no. Yes +tt0117731_shot_1067_img_0.jpg Is the actor inside the red bounding box called Jenny Wilson? Please answer yes or no. No +tt0118548_shot_1296_img_0.jpg Is the actor inside the red bounding box called Clint Eastwood? Please answer yes or no. Yes +tt0118548_shot_1296_img_0.jpg Is the actor inside the red bounding box called Kate Winslet? Please answer yes or no. No +tt0118571_shot_0627_img_0.jpg Is the actor inside the red bounding box called Glenn Close? Please answer yes or no. Yes +tt0118571_shot_0627_img_0.jpg Is the actor inside the red bounding box called Arlene Farber? Please answer yes or no. No +tt0118636_shot_0007_img_1.jpg Is the person inside the red bounding box called Brad Renfro? Please answer yes or no. Yes +tt0118636_shot_0007_img_1.jpg Is the person inside the red bounding box called Sandra Park? Please answer yes or no. No +tt0118636_shot_0344_img_0.jpg Is the actor inside the red bounding box called Brad Renfro? Please answer yes or no. Yes +tt0118636_shot_0344_img_0.jpg Is the actor inside the red bounding box called Karen Strassman? Please answer yes or no. No +tt0118655_shot_0279_img_0.jpg Is the person inside the red bounding box called Robert Wagner? Please answer yes or no. Yes +tt0118655_shot_0279_img_0.jpg Is the person inside the red bounding box called Arthur Birnbaum? Please answer yes or no. No +tt0118655_shot_1152_img_2.jpg Is the actor inside the red bounding box called Seth Green? Please answer yes or no. Yes +tt0118655_shot_1152_img_2.jpg Is the actor inside the red bounding box called Sue Doucette? Please answer yes or no. No +tt0118689_shot_0706_img_0.jpg Is the actor inside the red bounding box called Rowan Atkinson? Please answer yes or no. Yes +tt0118689_shot_0706_img_0.jpg Is the actor inside the red bounding box called Hugo Perez? Please answer yes or no. No +tt0118689_shot_0969_img_2.jpg Is the actor inside the red bounding box called Rowan Atkinson? Please answer yes or no. Yes +tt0118689_shot_0969_img_2.jpg Is the actor inside the red bounding box called Jack Shields? Please answer yes or no. No +tt0118715_shot_0079_img_0.jpg Is the actor inside the red bounding box called Jeff Bridges? Please answer yes or no. Yes +tt0118715_shot_0079_img_0.jpg Is the actor inside the red bounding box called Scott Adkins? Please answer yes or no. No +tt0118749_shot_0795_img_0.jpg Is the person inside the red bounding box called John C. Reilly? Please answer yes or no. Yes +tt0118749_shot_0795_img_0.jpg Is the person inside the red bounding box called Chris Lowell? Please answer yes or no. No +tt0118883_shot_0691_img_1.jpg Is the actor inside the red bounding box called Julia Roberts? Please answer yes or no. Yes +tt0118883_shot_0691_img_1.jpg Is the actor inside the red bounding box called Roger Bart? Please answer yes or no. No +tt0118971_shot_0679_img_0.jpg Is the actor inside the red bounding box called Charlize Theron? Please answer yes or no. Yes +tt0118971_shot_0679_img_0.jpg Is the actor inside the red bounding box called Young-min Kim? Please answer yes or no. No +tt0119008_shot_0979_img_0.jpg Is the actor inside the red bounding box named Al Pacino? Please answer yes or no. Yes +tt0119008_shot_0979_img_0.jpg Is the actor inside the red bounding box named Neil Tweddle? Please answer yes or no. No +tt0119094_shot_0446_img_2.jpg Is the actor inside the red bounding box called Nicolas Cage? Please answer yes or no. Yes +tt0119094_shot_0446_img_2.jpg Is the actor inside the red bounding box called Juan Gabriel Pareja? Please answer yes or no. No +tt0119116_shot_0721_img_0.jpg Is the actor inside the red bounding box called Bruce Willis? Please answer yes or no. Yes +tt0119116_shot_0721_img_0.jpg Is the actor inside the red bounding box called Troye Sivan? Please answer yes or no. No +tt0119174_shot_0439_img_0.jpg Is the actor inside the red bounding box named Michael Douglas? Please answer yes or no. Yes +tt0119174_shot_0439_img_0.jpg Is the actor inside the red bounding box named Carola McGuinness? Please answer yes or no. No +tt0119314_shot_0572_img_0.jpg Is the actor inside the red bounding box called Scarlett Johansson? Please answer yes or no. Yes +tt0119314_shot_0572_img_0.jpg Is the actor inside the red bounding box called Daisy Beaumont? Please answer yes or no. No +tt0119528_shot_0171_img_0.jpg Is the person inside the red bounding box called Jim Carrey? Please answer yes or no. Yes +tt0119528_shot_0171_img_0.jpg Is the person inside the red bounding box called Eliot Paton? Please answer yes or no. No +tt0119528_shot_0761_img_1.jpg Is the actor inside the red bounding box named Jim Carrey? Please answer yes or no. Yes +tt0119528_shot_0761_img_1.jpg Is the actor inside the red bounding box named Jari Kinnunen? Please answer yes or no. No +tt0119643_shot_0330_img_0.jpg Is the actor inside the red bounding box named Brad Pitt? Please answer yes or no. Yes +tt0119643_shot_0330_img_0.jpg Is the actor inside the red bounding box named Anthony Hopkins? Please answer yes or no. No +tt0119738_shot_0201_img_0.jpg Is the person inside the red bounding box named Christopher Masterson? Please answer yes or no. Yes +tt0119738_shot_0201_img_0.jpg Is the person inside the red bounding box named Edwin Craig? Please answer yes or no. No +tt0119822_shot_0878_img_0.jpg Is the person inside the red bounding box named Greg Kinnear? Please answer yes or no. Yes +tt0119822_shot_0878_img_0.jpg Is the person inside the red bounding box named Aleksandr Dubina? Please answer yes or no. No +tt0120338_shot_0444_img_2.jpg Is the actor inside the red bounding box named Kate Winslet? Please answer yes or no. Yes +tt0120338_shot_0444_img_2.jpg Is the actor inside the red bounding box named Donald Gibb? Please answer yes or no. No +tt0120338_shot_1130_img_2.jpg Is the person inside the red bounding box called Leonardo DiCaprio? Please answer yes or no. Yes +tt0120338_shot_1130_img_2.jpg Is the person inside the red bounding box called Anne Betancourt? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/code_reasoning.txt b/internvl_chat_audio/eval/mme/Your_Results/code_reasoning.txt new file mode 100644 index 00000000..f849f62d --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/code_reasoning.txt @@ -0,0 +1,40 @@ +0001.png The image shows a python code. Is the output of the code 'Hello'? Please answer yes or no. Yes +0001.png The image shows a python code. Is the output of the code 'World'? Please answer yes or no. No +0002.png The image shows a python code. Is the output of the code 'a cat'? Please answer yes or no. Yes +0002.png The image shows a python code. Is the output of the code 'a dog'? Please answer yes or no. No +0003.png The image shows a python code. Is the output of the code '12'? Please answer yes or no. Yes +0003.png The image shows a python code. Is the output of the code '5'? Please answer yes or no. No +0004.png The image shows a python code. Is the output of the code '3'? Please answer yes or no. Yes +0004.png The image shows a python code. Is the output of the code '2'? Please answer yes or no. No +0005.png The image shows a python code. Is the output of the code '12'? Please answer yes or no. Yes +0005.png The image shows a python code. Is the output of the code '5'? Please answer yes or no. No +0006.png The image shows a python code. Is the output of the code '0'? Please answer yes or no. Yes +0006.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. No +0007.png Is a c++ code shown in the picture? Please answer yes or no. Yes +0007.png Is a python code shown in the picture? Please answer yes or no. No +0008.png The image shows a python code. Is the output of the code '1234'? Please answer yes or no. Yes +0008.png The image shows a python code. Is the output of the code '12345'? Please answer yes or no. No +0009.png The image shows a python code. Is the output of the code '36'? Please answer yes or no. Yes +0009.png The image shows a python code. Is the output of the code '6'? Please answer yes or no. No +0010.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. Yes +0010.png The image shows a python code. Is the output of the code '5'? Please answer yes or no. No +0011.png The image shows a python code. Is the output of the code '0'? Please answer yes or no. Yes +0011.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. No +0012.png The image shows a python code. Is the output of the code 'working hard'? Please answer yes or no. Yes +0012.png The image shows a python code. Is the output of the code 'playing hard'? Please answer yes or no. No +0013.png The image shows a python code. Is the output of the code 'a cat'? Please answer yes or no. Yes +0013.png The image shows a python code. Is the output of the code 'a dog'? Please answer yes or no. No +0014.png The image shows a python code. Is the output of the code '7'? Please answer yes or no. Yes +0014.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. No +0015.png The image shows a python code. Is the output of the code '11'? Please answer yes or no. Yes +0015.png The image shows a python code. Is the output of the code '9'? Please answer yes or no. No +0016.png The image shows a python code. Is the output of the code 'x is smaller than 10'? Please answer yes or no. Yes +0016.png The image shows a python code. Is the output of the code 'x is larger than 10'? Please answer yes or no. No +0017.png The image shows a python code. Will the number 3 appear in the output of the code? Please answer yes or no. Yes +0017.png The image shows a python code. Will the number 6 appear in the output of the code? Please answer yes or no. No +0018.png The image shows a python code. Is the output of the code '11'? Please answer yes or no. Yes +0018.png The image shows a python code. Is the output of the code '12'? Please answer yes or no. No +0019.png The image shows a python code. Is the output of the code 'the list has more than 2 numbers'? Please answer yes or no. Yes +0019.png The image shows a python code. Is the output of the code 'the list has less than 2 numbers'? Please answer yes or no. No +0020.png Is a python code shown in the picture? Please answer yes or no. Yes +0020.png Is a c++ code shown in the picture? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/color.txt b/internvl_chat_audio/eval/mme/Your_Results/color.txt new file mode 100644 index 00000000..f3cd836d --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/color.txt @@ -0,0 +1,60 @@ +000000006723.jpg Is there a red brick building in the image? Please answer yes or no. Yes +000000006723.jpg Is there a yellow brick building in the image? Please answer yes or no. No +000000008277.jpg Is there a white plate in the image? Please answer yes or no. Yes +000000008277.jpg Is there a yellow plate in the image? Please answer yes or no. No +000000012120.jpg Is there a blue court in the image? Please answer yes or no. Yes +000000012120.jpg Is there a purple court in the image? Please answer yes or no. No +000000014831.jpg Is there a brown and white animal in the image? Please answer yes or no. Yes +000000014831.jpg Is there a green and red animal in the image? Please answer yes or no. No +000000028993.jpg Are there yellow poles in the image? Please answer yes or no. Yes +000000028993.jpg Are there blue poles in the image? Please answer yes or no. No +000000029393.jpg Is there a brown dog in the image? Please answer yes or no. Yes +000000029393.jpg Is there a black dog in the image? Please answer yes or no. No +000000035770.jpg Is there a black and white toilet in the image? Please answer yes or no. Yes +000000035770.jpg Is there a red and white toilet in the image? Please answer yes or no. No +000000038118.jpg Is there a red coat in the image? Please answer yes or no. Yes +000000038118.jpg Is there a yellow coat in the image? Please answer yes or no. No +000000047112.jpg Is there a white plate in the image? Please answer yes or no. Yes +000000047112.jpg Is there a yellow plate in the image? Please answer yes or no. No +000000047121.jpg Is there a black cat in the image? Please answer yes or no. Yes +000000047121.jpg Is there a brown cat in the image? Please answer yes or no. No +000000053529.jpg Is there a green hat in the image? Please answer yes or no. Yes +000000053529.jpg Is there a red hat in the image? Please answer yes or no. No +000000053994.jpg Is there a gray wall in the image? Please answer yes or no. Yes +000000053994.jpg Is there a red wall in the image? Please answer yes or no. No +000000055072.jpg Is there a brown giraffe in the image? Please answer yes or no. Yes +000000055072.jpg Is there a black giraffe in the image? Please answer yes or no. No +000000057597.jpg Are there any red shoes in the image? Please answer yes or no. Yes +000000057597.jpg Are there any yellow shoes in the image? Please answer yes or no. No +000000061658.jpg Are there a white dish in the image? Please answer yes or no. Yes +000000061658.jpg Are there a green dish in the image? Please answer yes or no. No +000000338560.jpg Is there a blue and yellow fire hydrant in the image? Please answer yes or no. Yes +000000338560.jpg Is there a blue and orange fire hydrant in the image? Please answer yes or no. No +000000370208.jpg Is there a red bicycle with white handlebars in the image? Please answer yes or no. Yes +000000370208.jpg Is there a red bicycle with black handlebars in the image? Please answer yes or no. No +000000377723.jpg Is there a blue bus in the image? Please answer yes or no. Yes +000000377723.jpg Is there a orange bus in the image? Please answer yes or no. No +000000405205.jpg Is there a white bus in the image? Please answer yes or no. Yes +000000405205.jpg Is there a red bus in the image? Please answer yes or no. No +000000410612.jpg Is there a red boat in the image? Please answer yes or no. Yes +000000410612.jpg Is there a gray boat in the image? Please answer yes or no. No +000000427034.jpg Is there a brown and black dog in the image? Please answer yes or no. Yes +000000427034.jpg Is there a brown and white dog in the image? Please answer yes or no. No +000000442456.jpg Is there a man wearing a red shirt in the image? Please answer yes or no. Yes +000000442456.jpg Is there a man wearing a white shirt in the image? Please answer yes or no. No +000000492362.jpg Is there a skateboard with red wheels in the image? Please answer yes or no. Yes +000000492362.jpg Is there a skateboard with black wheels in the image? Please answer yes or no. No +000000492992.jpg Is there a white bird in the image? Please answer yes or no. Yes +000000492992.jpg Is there a yellow bird in the image? Please answer yes or no. No +000000512929.jpg Are there any green beans in the image? Please answer yes or no. Yes +000000512929.jpg Are there any orange beans in the image? Please answer yes or no. No +000000530457.jpg Are there any red flowers in the image? Please answer yes or no. Yes +000000530457.jpg Are there any green flowers in the image? Please answer yes or no. No +000000532761.jpg Is there a living room painted yellow in the image? Please answer yes or no. Yes +000000532761.jpg Is there a living room painted black in the image? Please answer yes or no. No +000000534041.jpg Is there a purple bottle in the image? Please answer yes or no. Yes +000000534041.jpg Is there a white bottle in the image? Please answer yes or no. No +000000563758.jpg Is there a red scarf in the image? Please answer yes or no. Yes +000000563758.jpg Is there a brown scarf in the image? Please answer yes or no. No +000000564280.jpg Is there a red couch in the image? Please answer yes or no. Yes +000000564280.jpg Is there a black couch in the image? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/commonsense_reasoning.txt b/internvl_chat_audio/eval/mme/Your_Results/commonsense_reasoning.txt new file mode 100644 index 00000000..de3af0fc --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/commonsense_reasoning.txt @@ -0,0 +1,140 @@ +0001.png Here are the order details for my taxi ride. Should I actually pay $29.42? Please answer yes or no. Yes +0001.png Here are the order details for my taxi ride. Should I actually pay $32.42? Please answer yes or no. No +0002.png Should I stop when I'm about to cross the street and see the sign in the picture? Please answer yes or no. Yes +0002.png When I see the sign in the picture, can I cross the street? Please answer yes or no. No +0003.png May I ask if in the game of finger-guessing game, did the right side of the picture win? Please answer yes or no. Yes +0003.png May I ask if in the game of finger-guessing game, did the left side of the picture win? Please answer yes or no. No +0004.png Does the fruit in the picture look stale? Please answer yes or no. Yes +0004.png Does the fruit in the picture look very fresh? Please answer yes or no. No +0005.png The office's normal closing time is 5 p.m. Now is afternoon. Should I continue to work at the time shown in the picture? Please answer yes or no. Yes +0005.png The office's normal closing time is 5 p.m. Now is afternoon. Could I leave work at the time shown in the picture? Please answer yes or no. No +0006.png I recently want to go on vacation to relax and go to a place full of fresh air. Is the venue in the picture appropriate? Please answer yes or no. Yes +0006.png I want to go where there are a lot of people. Is the venue in the picture appropriate? Please answer yes or no. No +0007.png I want to clean the house and I want to choose a tool. Is the tool in the picture an appropriate choice? Please answer yes or no. Yes +0007.png I want to transport something and I want to choose a tool to help me. Is the tool in the picture an appropriate choice? Please answer yes or no. No +0008.png Can I smoke where the picture is? Please answer yes or no. Yes +0008.png Is smoking prohibited in the location of the picture? Please answer yes or no. No +0009.png Will green be obtained by mixing the above two colors? Please answer yes or no. Yes +0009.png Will red be obtained by mixing the above two colors? Please answer yes or no. No +0010.png I am going to exercise and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. Yes +0010.png I am going to study and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. No +0011.png If I am allergic to durian, can I finish the fruit in the picture? Please answer yes or no. Yes +0011.png If I am allergic to banana, can I finish the fruit in the picture? Please answer yes or no. No +0012.png I am going to study and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. Yes +0012.png I am going to exercise and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. No +0013.png I am going to a formal dinner party. Is the shoe in the picture an appropriate choice? Please answer yes or no. Yes +0013.png I am going to play basketball. Is the shoe in the picture an appropriate choice? Please answer yes or no. No +0014.png In this line chart, the vertical axis is height and the horizontal axis is age. Does Maria's height exceed Jane's height in the end? Please answer yes or no. Yes +0014.png In this line chart, the vertical axis is height and the horizontal axis is age. Does Jane's height exceed Kangkang's height in the end? Please answer yes or no. No +0015.png Is the ball usually played with hands? Please answer yes or no. Yes +0015.png Is the ball usually played with feet? Please answer yes or no. No +0016.png Is the place in the picture a good place to enjoy the cool in a sunny day? Please answer yes or no. Yes +0016.png Is the place in the picture a good shelter from the rain when it thunders outside? Please answer yes or no. No +0017.png Are the vehicles in the pictures usually environmentally friendly? Please answer yes or no. Yes +0017.png Does the vehicle in the picture usually run faster than the car? Please answer yes or no. No +0018.png This is a picture of some kind of animal. Does it eat leaves? Please answer yes or no. Yes +0018.png This is a picture of some kind of animal. Does it eat meat? Please answer yes or no. No +0019.png Is the water flow in the picture from the top to the bottom? Please answer yes or no. Yes +0019.png Is the water flow in the picture from the bottom to the top? Please answer yes or no. No +0020.png Can the item in the picture be used to measure length? Please answer yes or no. Yes +0020.png Can the item in the picture be used to measure angles? Please answer yes or no. No +0021.png This is a toilet guide sign. I am a man. Should I go to the toilet on the left? Please answer yes or no. Yes +0021.png This is a toilet guide sign. I am a man. Should I go to the toilet on the right? Please answer yes or no. No +0022.png Does the animal in the picture usually catch mice? Please answer yes or no. Yes +0022.png Is the animal in the picture usually used in search and rescue? Please answer yes or no. No +0023.png If you want to keep your fruit fresh in summer, should you put it in the appliance in the picture? Please answer yes or no. Yes +0023.png Is the appliance in the picture more suitable for winter than summer? Please answer yes or no. No +0024.png I want to go skating. Is the shoe in the picture usually appropriate? Please answer yes or no. Yes +0024.png I want to go roller skating. Is the shoe in the picture usually appropriate? Please answer yes or no. No +0025.png I feel very thirsty in the desert now. Can the thing in the picture help me? Please answer yes or no. Yes +0025.png I don't like clear cups. Is the cup in the picture my type? Please answer yes or no. No +0026.png I want to go for a run and I want to choose a pair of shoes. Is the shoe in the picture an appropriate choice? Please answer yes or no. Yes +0026.png I want to practice ballet and I want to choose a pair of shoes. Is the shoe in the picture an appropriate choice? Please answer yes or no. No +0027.png Are the pants in the picture usually suitable for casual wear? Please answer yes or no. Yes +0027.png Are the pants in the picture usually suitable for playing basketball? Please answer yes or no. No +0028.png This is a picture from a real scene. Is there only one real cat in this picture? Please answer yes or no. Yes +0028.png This is a picture from a real scene. Is there only two real cats in this picture? Please answer yes or no. No +0029.png The three cats in the picture, the one without a beard, is the middle one? Please answer yes or no. Yes +0029.png The three cats in the picture, the one without a beard, is the right one? Please answer yes or no. No +0030.png I'm going to 501. Do I need to turn left at the intersection? Please answer yes or no. Yes +0030.png I'm going to 502. Do I need to turn left at the intersection? Please answer yes or no. No +0031.png Is the drink in the picture usually suitable for a party? Please answer yes or no. Yes +0031.png Is the drink in the picture usually suitable for drinking together with cephalosporin? Please answer yes or no. No +0032.png Here is a picture of the cake I cut. Did I cut it at least twice? Please answer yes or no. Yes +0032.png Here is a picture of the cake I cut. Did I cut it at least once? Please answer yes or no. No +0033.png This is a picture from a real scene. Is there only one real apple in this picture? Please answer yes or no. Yes +0033.png This is a picture from a real scene. Is there only two real apples in this picture? Please answer yes or no. No +0034.png Here is a pie chart counting the favorite fruits of all employees in our company. Is the durian the most popular fruit? Please answer yes or no. Yes +0034.png Here is a pie chart counting the favorite fruits of all employees in our company. Is the mango the most popular fruit? Please answer yes or no. No +0035.png This is the sales chart of this month. Is Tina the runner-up in sales this month? Please answer yes or no. Yes +0035.png This is the sales chart of this month. Is John the runner-up in sales this month? Please answer yes or no. No +0036.png Is it a good time to walk through the road in the picture? Please answer yes or no. Yes +0036.png Is it a good time to drive a car through the road in the picture? Please answer yes or no. No +0037.png Here is a photo of the sun's position at a certain time. Could it be dusk now? Please answer yes or no. Yes +0037.png Here is a photo of the sun's position at a certain time. Could it be noon now? Please answer yes or no. No +0038.png All apples are shown in the picture. If I eat an apple every day, can I eat it for four days? Please answer yes or no. Yes +0038.png All apples are shown in the picture. If I eat an apple every day, can I eat it for three days? Please answer yes or no. No +0039.png This line chart is used to count the sales of two types of burgers. Are chicken burgers more popular? Please answer yes or no. Yes +0039.png This line chart is used to count the sales of two types of burgers. Are beef burgers more popular? Please answer yes or no. No +0040.png I want to supplement protein. Is it appropriate to eat the food in the picture? Please answer yes or no. Yes +0040.png I don't like to eat any food related to chicken. Is the food in the picture my type? Please answer yes or no. No +0041.png Is the fruit in the picture usually sweet? Please answer yes or no. Yes +0041.png Is the fruit in the picture usually spicy? Please answer yes or no. No +0042.png Are there usually cars in the area shown in the picture? Please answer yes or no. Yes +0042.png Is it appropriate to cross the road directly from the place shown in the picture? Please answer yes or no. No +0043.png Is the animal in the picture usually not seen in winter? Please answer yes or no. Yes +0043.png Is the animal in the picture usually seen in winter? Please answer yes or no. No +0044.png This is a flowchart of a program. I enter 3 and 6. Is the output 'No'? Please answer yes or no. Yes +0044.png This is a flowchart of a program. I enter 3 and 6. Is the output 'Yes'? Please answer yes or no. No +0045.png There is a sign at the intersection, can I turn left? Please answer yes or no. Yes +0045.png There is a sign at the intersection, can I turn right? Please answer yes or no. No +0046.png Vitamin C is very helpful for human health. Does the food on in the picture usually contain Vitamin C? Please answer yes or no. Yes +0046.png Is the food in the picture commonly used to build muscle? Please answer yes or no. No +0047.png All apples are shown in the picture. My brother and I divide the apples equally. May I have one apple? Please answer yes or no. Yes +0047.png All apples are shown in the picture. My brother and I divide the apples equally. May I have two apples? Please answer yes or no. No +0048.png Here is a picture of eating fruit. Am I eating a strawberry? Please answer yes or no. Yes +0048.png Here is a picture of eating fruit. Am I eating a cherry tomato? Please answer yes or no. No +0049.png Does the vehicle in the picture usually have its Windows closed during fast driving? Please answer yes or no. Yes +0049.png Does the vehicle in the picture usually have its Windows opened during fast driving? Please answer yes or no. No +0050.png Do people commonly use the item in the picture for makeup in their daily lives? Please answer yes or no. Yes +0050.png Do people commonly use the item in the picture to write in their daily lives? Please answer yes or no. No +0051.png This is a flowchart of a program. When the input is 5, is the output 6? Please answer yes or no. Yes +0051.png This is a flowchart of a program. When the input is 6, is the output 5? Please answer yes or no. No +0052.png I want to lose weight. Is the food in the picture an appropriate choice? Please answer yes or no. Yes +0052.png I want to gain weight. Is the food in the picture an appropriate choice? Please answer yes or no. No +0053.png Is the car in the picture going to make a right turn after going through a straight road section? Please answer yes or no. Yes +0053.png Is the car in the picture going to make a left turn after going through a straight road section? Please answer yes or no. No +0054.png May I ask if the plants in the picture can survive in the water? Please answer yes or no. Yes +0054.png May I ask if the plants in the picture can survive in the soil? Please answer yes or no. No +0055.png The man in the picture is eating. Does he eat noodles? Please answer yes or no. Yes +0055.png The man in the picture is eating. Does he eat rice? Please answer yes or no. No +0056.png Can the item in the picture output water? Please answer yes or no. Yes +0056.png Can the item in picture be used for blowing air? Please answer yes or no. No +0057.png Does the vehicle in the picture usually run faster than a horse? Please answer yes or no. Yes +0057.png Does the vehicle in the picture usually fly? Please answer yes or no. No +0058.png Can't I smoke here? Please answer yes or no. Yes +0058.png May I smoke here? Please answer yes or no. No +0059.png This pie chart is the age distribution of our company. Is the proportion of people aged 30-50 more than 40%? Please answer yes or no. Yes +0059.png This pie chart is the age distribution of our company. Is the proportion of people aged 40-50 more than 30%? Please answer yes or no. No +0060.png This is the histogram of fruit sales today. Do more men buy watermelons than women buy bananas? Please answer yes or no. Yes +0060.png This is the histogram of fruit sales today. Do more men buy peach than women buy apple? Please answer yes or no. No +0061.png Is the tool in the picture common in tall buildings? Please answer yes or no. Yes +0061.png In case of fire, is it appropriate to choose the tool in the picture to go downstairs? Please answer yes or no. No +0062.png It's snowing outside the window now. I want to go out. Is it appropriate to wear the cloth in the picture? Please answer yes or no. Yes +0062.png It's very hot outside. I want to go out. Is it appropriate to wear the cloth in the picture? Please answer yes or no. No +0063.png Is the animal in the picture suitable as a pet? Please answer yes or no. Yes +0063.png Is the animal in the pictures usually stronger than adult tigers? Please answer yes or no. No +0064.png I want to play basketball. Is the venue in the picture a good choice? Please answer yes or no. Yes +0064.png I want to play football. Is the venue in the picture a good choice? Please answer yes or no. No +0065.png Is it appropriate to wear a down jacket during the season in the picture? Please answer yes or no. Yes +0065.png Is it appropriate to only wear short sleeves during the season in the picture? Please answer yes or no. No +0066.png I want to carry one thing with me on a rainy day. Is the thing in the image an appropriate choice? Please answer yes or no. Yes +0066.png It is raining outside. I am in a house and I don't need to go out. Is this thing in the picture necessary for me to use? Please answer yes or no. No +0067.png I feel very hot. Is the tool in the picture suitable for use? Please answer yes or no. Yes +0067.png I feel very cold. Is the tool in the picture suitable for use? Please answer yes or no. No +0068.png Is it unhealthy to eat the food in the picture too often? Please answer yes or no. Yes +0068.png Is the food in the picture usually low in calories? Please answer yes or no. No +0069.png Is the phone in the photo connected to a charger? Please answer yes or no. Yes +0069.png Is the phone in the photo charging? Please answer yes or no. No +0070.png I want to turn the screw. Is the tool in the picture usually appropriate? Please answer yes or no. Yes +0070.png Is the tool in the picture usually suitable for smashing walnuts? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/count.txt b/internvl_chat_audio/eval/mme/Your_Results/count.txt new file mode 100644 index 00000000..986e6f89 --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/count.txt @@ -0,0 +1,60 @@ +000000006040.jpg Is there a train in the picture? Please answer yes or no. Yes +000000006040.jpg Are there a total of two trains in the picture? Please answer yes or no. No +000000044279.jpg Is there a total of two people in the image? Please answer yes or no. Yes +000000044279.jpg Is there only one people in the image? Please answer yes or no. No +000000067213.jpg Is there only one dog in the image? Please answer yes or no. Yes +000000067213.jpg Is there two dogs in the image? Please answer yes or no. No +000000071226.jpg Is there a total of two dogs in the image? Please answer yes or no. Yes +000000071226.jpg Is there only one dogs in the image? Please answer yes or no. No +000000097994.jpg Are there three laptops in the picture? Please answer yes or no. Yes +000000097994.jpg Are there four laptops in the picture? Please answer yes or no. No +000000195918.jpg Is there a total of two display devices in the image? Please answer yes or no. Yes +000000195918.jpg Is there only one display device in the image? Please answer yes or no. No +000000236721.jpg Are there two bananas in the image? Please answer yes or no. Yes +000000236721.jpg Are there three bananas in the image? Please answer yes or no. No +000000261712.jpg Are there two giraffes in this image? Please answer yes or no. Yes +000000261712.jpg Are there three giraffes in this picture? Please answer yes or no. No +000000274066.jpg Are there four people appear in this image? Please answer yes or no. Yes +000000274066.jpg Are there only three people appear in this image? Please answer yes or no. No +000000276434.jpg Is there a total of three cakes in this image? Please answer yes or no. Yes +000000276434.jpg Are there only two cakes in this image? Please answer yes or no. No +000000289059.jpg Is there a total of two person appear in the image? Please answer yes or no. Yes +000000289059.jpg Is there only one person appear in the image? Please answer yes or no. No +000000290081.jpg Is there only one bowl in this image? Please answer yes or no. Yes +000000290081.jpg Are there two bowls in this image? Please answer yes or no. No +000000301867.jpg Are there three people appear in this image? Please answer yes or no. Yes +000000301867.jpg Are there only two people appear in this image? Please answer yes or no. No +000000335954.jpg Are there two bowls in this image? Please answer yes or no. Yes +000000335954.jpg Are there three bowls in this image? Please answer yes or no. No +000000357816.jpg Are there four people in this image? Please answer yes or no. Yes +000000357816.jpg Are there five people in this image? Please answer yes or no. No +000000372819.jpg Are there four dogs appear in this image? Please answer yes or no. Yes +000000372819.jpg Are there only three dogs appear in this image? Please answer yes or no. No +000000410612.jpg Is there only one ship in the picture? Please answer yes or no. Yes +000000410612.jpg Is there a total of two ships in the picture? Please answer yes or no. No +000000423944.jpg Is there no person in this picture? Please answer yes or no. Yes +000000423944.jpg Are there two people appear in this image? Please answer yes or no. No +000000427034.jpg Is there a dog in the picture? Please answer yes or no. Yes +000000427034.jpg Are there a total of two dogs in the picture? Please answer yes or no. No +000000430286.jpg Are there three remotes in this image? Please answer yes or no. Yes +000000430286.jpg Are there only two remotes in this image? Please answer yes or no. No +000000432468.jpg Are there three zippers in the picture? Please answer yes or no. Yes +000000432468.jpg Is there a zipper in the picture? Please answer yes or no. No +000000434479.jpg Are there two pieces of pizza in this image? Please answer yes or no. Yes +000000434479.jpg Is there only one piece of pizza in this image? Please answer yes or no. No +000000438304.jpg Are there two tennis rackets in the picture? Please answer yes or no. Yes +000000438304.jpg Are there only one tennis racket in the picture? Please answer yes or no. No +000000450303.jpg Are there six people appear in this image? Please answer yes or no. Yes +000000450303.jpg Are there seven people appear in this image? Please answer yes or no. No +000000470121.jpg Is there only one bottle in the image? Please answer yes or no. Yes +000000470121.jpg Is there two bottles in the image? Please answer yes or no. No +000000476215.jpg Are there two horses in this image? Please answer yes or no. Yes +000000476215.jpg Is there only one horse in this image? Please answer yes or no. No +000000482100.jpg Are there two toilets in the picture? Please answer yes or no. Yes +000000482100.jpg Is there only one toilet in the picture? Please answer yes or no. No +000000491867.jpg Is there only one necktie in the image? Please answer yes or no. Yes +000000491867.jpg Is there three neckties in the image? Please answer yes or no. No +000000556000.jpg Are there four people in the image? Please answer yes or no. Yes +000000556000.jpg Are there only three people in the image? Please answer yes or no. No +000000565045.jpg Are there two bath towels in the picture? Please answer yes or no. Yes +000000565045.jpg Is there only one bath towel in the picture? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/existence.txt b/internvl_chat_audio/eval/mme/Your_Results/existence.txt new file mode 100644 index 00000000..c3da2b48 --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/existence.txt @@ -0,0 +1,60 @@ +000000006040.jpg Is there a train in this image? Please answer yes or no. Yes +000000006040.jpg Is there a bed in this image? Please answer yes or no. No +000000006471.jpg Is there a baseball bat in this image? Please answer yes or no. Yes +000000006471.jpg Is there a giraffe in this image? Please answer yes or no. No +000000007108.jpg Is there a elephant in this image? Please answer yes or no. Yes +000000007108.jpg Is there a hair drier in this image? Please answer yes or no. No +000000007816.jpg Is there a motorcycle in this image? Please answer yes or no. Yes +000000007816.jpg Is there a airplane in this image? Please answer yes or no. No +000000007977.jpg Is there a skateboard in this image? Please answer yes or no. Yes +000000007977.jpg Is there a spoon in this image? Please answer yes or no. No +000000008844.jpg Is there a person in this image? Please answer yes or no. Yes +000000008844.jpg Is there a sink in this image? Please answer yes or no. No +000000009590.jpg Is there a bottle in this image? Please answer yes or no. Yes +000000009590.jpg Is there a scissors in this image? Please answer yes or no. No +000000010363.jpg Is there a bottle in this image? Please answer yes or no. Yes +000000010363.jpg Is there a apple in this image? Please answer yes or no. No +000000011197.jpg Is there a car in this image? Please answer yes or no. Yes +000000011197.jpg Is there a fork in this image? Please answer yes or no. No +000000015254.jpg Is there a spoon in this image? Please answer yes or no. Yes +000000015254.jpg Is there a donut in this image? Please answer yes or no. No +000000015517.jpg Is there a bus in this image? Please answer yes or no. Yes +000000015517.jpg Is there a cow in this image? Please answer yes or no. No +000000015746.jpg Is there a fire hydrant in this image? Please answer yes or no. Yes +000000015746.jpg Is there a person in this image? Please answer yes or no. No +000000037751.jpg Is there a backpack in this image? Please answer yes or no. Yes +000000037751.jpg Is there a microwave in this image? Please answer yes or no. No +000000050145.jpg Is there a bicycle in this image? Please answer yes or no. Yes +000000050145.jpg Is there a apple in this image? Please answer yes or no. No +000000061418.jpg Is there a chair in this image? Please answer yes or no. Yes +000000061418.jpg Is there a airplane in this image? Please answer yes or no. No +000000417779.jpg Is there a car in this image? Please answer yes or no. Yes +000000417779.jpg Is there a kite in this image? Please answer yes or no. No +000000424521.jpg Is there a skateboard in this image? Please answer yes or no. Yes +000000424521.jpg Is there a banana in this image? Please answer yes or no. No +000000438304.jpg Is there a sports ball in this image? Please answer yes or no. Yes +000000438304.jpg Is there a horse in this image? Please answer yes or no. No +000000494427.jpg Is there a laptop in this image? Please answer yes or no. Yes +000000494427.jpg Is there a potted plant in this image? Please answer yes or no. No +000000495448.jpg Is there a cake in this image? Please answer yes or no. Yes +000000495448.jpg Is there a tie in this image? Please answer yes or no. No +000000498463.jpg Is there a refrigerator in this image? Please answer yes or no. Yes +000000498463.jpg Is there a donut in this image? Please answer yes or no. No +000000519039.jpg Is there a truck in this image? Please answer yes or no. Yes +000000519039.jpg Is there a book in this image? Please answer yes or no. No +000000523241.jpg Is there a car in this image? Please answer yes or no. Yes +000000523241.jpg Is there a cell phone in this image? Please answer yes or no. No +000000530162.jpg Is there a umbrella in this image? Please answer yes or no. Yes +000000530162.jpg Is there a horse in this image? Please answer yes or no. No +000000537812.jpg Is there a chair in this image? Please answer yes or no. Yes +000000537812.jpg Is there a baseball bat in this image? Please answer yes or no. No +000000541952.jpg Is there a clock in this image? Please answer yes or no. Yes +000000541952.jpg Is there a bottle in this image? Please answer yes or no. No +000000546626.jpg Is there a bottle in this image? Please answer yes or no. Yes +000000546626.jpg Is there a mouse in this image? Please answer yes or no. No +000000556000.jpg Is there a chair in this image? Please answer yes or no. Yes +000000556000.jpg Is there a dog in this image? Please answer yes or no. No +000000557258.jpg Is there a toilet in this image? Please answer yes or no. Yes +000000557258.jpg Is there a pizza in this image? Please answer yes or no. No +000000572956.jpg Is there a motorcycle in this image? Please answer yes or no. Yes +000000572956.jpg Is there a bus in this image? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/landmark.txt b/internvl_chat_audio/eval/mme/Your_Results/landmark.txt new file mode 100644 index 00000000..1259cf4e --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/landmark.txt @@ -0,0 +1,400 @@ +0006adf999ccc899.jpg Is this a photo of Muktinath? Please answer yes or no. Yes +0006adf999ccc899.jpg Is this a photo of Lohikoski Church? Please answer yes or no. No +0015309638e5a9b1.jpg Is this a picture of Teufelsmauer (Harz)? Please answer yes or no. Yes +0015309638e5a9b1.jpg Is this a picture of Romfo kirke? Please answer yes or no. No +001f0bfe8464254a.jpg Is this a picture of Saint Catherine of Alexandria church in Warsaw? Please answer yes or no. Yes +001f0bfe8464254a.jpg Is this a picture of Kaap Skil? Please answer yes or no. No +0022666a36b80ea5.jpg Is this an image of Hall of Supreme Harmony (Forbidden City)? Please answer yes or no. Yes +0022666a36b80ea5.jpg Is this an image of Brickegickel? Please answer yes or no. No +0023bd687cbc0a44.jpg Is this a photo of Utrecht Dom Tower? Please answer yes or no. Yes +0023bd687cbc0a44.jpg Is this a photo of Great Palace Mosaic Museum? Please answer yes or no. No +00261324328490fc.jpg Is this an image of Loch Ossian? Please answer yes or no. Yes +00261324328490fc.jpg Is this an image of Tanemakidaishi? Please answer yes or no. No +002ae473a76362a6.jpg Is this an image of Gerlache Strait? Please answer yes or no. Yes +002ae473a76362a6.jpg Is this an image of Schmiedebruch (Naturschutzgebiet)? Please answer yes or no. No +003f8af9477a8eb8.jpg Is this a photo of Le Destroit? Please answer yes or no. Yes +003f8af9477a8eb8.jpg Is this a photo of Kharkov Mosque? Please answer yes or no. No +004543cec064cd0b.jpg Is this a picture of Wollaton Park? Please answer yes or no. Yes +004543cec064cd0b.jpg Is this a picture of St John's Church, Blackpool? Please answer yes or no. No +004beea6937b3a8c.jpg Is this a photo of Brussels Park? Please answer yes or no. Yes +004beea6937b3a8c.jpg Is this a photo of Forest Hills Cemetery (Boston, Massachusetts)? Please answer yes or no. No +005150f80de6f487.jpg Is this a photo of Schloss Burgpreppach? Please answer yes or no. Yes +005150f80de6f487.jpg Is this a photo of Pinkas Synagogue? Please answer yes or no. No +0053894534d1e259.jpg Is this a picture of Goethe- und Schiller-Denkmal (Weimar)? Please answer yes or no. Yes +0053894534d1e259.jpg Is this a picture of Grey River? Please answer yes or no. No +0059fc817c5fd5b7.jpg Is this a photo of KLCC Park? Please answer yes or no. Yes +0059fc817c5fd5b7.jpg Is this a photo of Neath Castle? Please answer yes or no. No +005c68530bb6ed7c.jpg Is this a photo of Balvenie Castle? Please answer yes or no. Yes +005c68530bb6ed7c.jpg Is this a photo of Draper Utah Temple? Please answer yes or no. No +006e3342b57aa3b4.jpg Is this a photo of Larvs kyrka? Please answer yes or no. Yes +006e3342b57aa3b4.jpg Is this a photo of Geitgallien? Please answer yes or no. No +0075a4d212539625.jpg Is this a photo of Cardiff Castle? Please answer yes or no. Yes +0075a4d212539625.jpg Is this a photo of Boudhanath? Please answer yes or no. No +008053e6bb5afa33.jpg Is this a photo of Rochers des Mourres? Please answer yes or no. Yes +008053e6bb5afa33.jpg Is this a photo of Schloss Weinfelden? Please answer yes or no. No +00c1349a9622390c.jpg Is this a photo of Amar Jawan Jyoti (Tomb of the Unknown Soldier)? Please answer yes or no. Yes +00c1349a9622390c.jpg Is this a photo of Porta Garibaldi (Marsala)? Please answer yes or no. No +00cae9f165d8d2cc.jpg Is this an image of Fisherman's Bastion? Please answer yes or no. Yes +00cae9f165d8d2cc.jpg Is this an image of Equestrian statue of George IV, Trafalgar Square, London? Please answer yes or no. No +00f4fb71575efb0b.jpg Is this a picture of 1965)? Please answer yes or no. Yes +00f4fb71575efb0b.jpg Is this a picture of Plaza Monumental de Toros de Pueblo Nuevo? Please answer yes or no. No +00fa476690a260a8.jpg Is this an image of Bathampton Toll Bridge? Please answer yes or no. Yes +00fa476690a260a8.jpg Is this an image of Moose River Plains Wild Forest? Please answer yes or no. No +010260e62e7d847d.jpg Is this a photo of Visingsborg? Please answer yes or no. Yes +010260e62e7d847d.jpg Is this a photo of Teutsche Schule (Schleusingen)? Please answer yes or no. No +010b5b5c9c461f65.jpg Is this an image of Het Steen? Please answer yes or no. Yes +010b5b5c9c461f65.jpg Is this an image of Castello di Montalto Dora? Please answer yes or no. No +01114538f95f1ce8.jpg Is this a picture of Svirzh Castle? Please answer yes or no. Yes +01114538f95f1ce8.jpg Is this a picture of Bistrica, Ionian Sea? Please answer yes or no. No +01121b9b3c79afc4.jpg Is this a photo of Alamannenmuseum Ellwangen? Please answer yes or no. Yes +01121b9b3c79afc4.jpg Is this a photo of Sint-Jozefkerk, Enschede? Please answer yes or no. No +011a19afda5df3c7.jpg Is this a picture of San Prospero (Reggio Emilia)? Please answer yes or no. Yes +011a19afda5df3c7.jpg Is this a picture of Lake DeFuniak? Please answer yes or no. No +012856462a95ada9.jpg Is this a picture of Chingshui Cliff? Please answer yes or no. Yes +012856462a95ada9.jpg Is this a picture of St. John's Church, Portland? Please answer yes or no. No +012ea376a16f17bf.jpg Is this a picture of Church of Saint Giles in Prague? Please answer yes or no. Yes +012ea376a16f17bf.jpg Is this a picture of Pfarrkirche St. Martin an der Raab? Please answer yes or no. No +013e1a1e887cb915.jpg Is this a picture of Real Casa de Correos, Madrid? Please answer yes or no. Yes +013e1a1e887cb915.jpg Is this a picture of Ayton Castle, Scottish Borders? Please answer yes or no. No +0140a7d5446719c3.jpg Is this a photo of Friday Mosque, Herat? Please answer yes or no. Yes +0140a7d5446719c3.jpg Is this a photo of Hexenturm (Burg (bei Magdeburg))? Please answer yes or no. No +01416124a7ab211a.jpg Is this a photo of Ajuntament de Mollerussa? Please answer yes or no. Yes +01416124a7ab211a.jpg Is this a photo of Episcopal Diocese of Northern Indiana? Please answer yes or no. No +01425c4d35880aad.jpg Is this an image of Museu de Sant Boi de Llobregat? Please answer yes or no. Yes +01425c4d35880aad.jpg Is this an image of Bezdonnoye Lake, Moscow Oblast? Please answer yes or no. No +014c50e91cf799c7.jpg Is this a photo of Clearwater Beach, Florida? Please answer yes or no. Yes +014c50e91cf799c7.jpg Is this a photo of Church of Holy Trinity in Vratislavice nad Nisou? Please answer yes or no. No +015543ddea466c54.jpg Is this a picture of Galician market square in Skansen, Sanok? Please answer yes or no. Yes +015543ddea466c54.jpg Is this a picture of Walworth Castle? Please answer yes or no. No +0169c4235270f65b.jpg Is this a photo of St. Ricarius Church, Aberford? Please answer yes or no. Yes +0169c4235270f65b.jpg Is this a photo of Schloss Prebberede? Please answer yes or no. No +016e68628ea6c7ee.jpg Is this an image of Jewish Ceremonial Hall, Prague-Josefov? Please answer yes or no. Yes +016e68628ea6c7ee.jpg Is this an image of Our Lady Church (Ettlingen)? Please answer yes or no. No +01862535ce8961ee.jpg Is this an image of Mont Tendre? Please answer yes or no. Yes +01862535ce8961ee.jpg Is this an image of Ogden Utah Temple? Please answer yes or no. No +0186742e7963fb3c.jpg Is this a photo of Ozegahara? Please answer yes or no. Yes +0186742e7963fb3c.jpg Is this a photo of San Giovenale (Orvieto)? Please answer yes or no. No +01928acd6dfe6cdd.jpg Is this a picture of Watergate Bay? Please answer yes or no. Yes +01928acd6dfe6cdd.jpg Is this a picture of Cairnpapple Hill? Please answer yes or no. No +01a6842099d62f22.jpg Is this an image of Clark Veterans Cemetery? Please answer yes or no. Yes +01a6842099d62f22.jpg Is this an image of Mirdif City Centre? Please answer yes or no. No +01aca2d30b43d05f.jpg Is this a picture of Monte Minerva (Villanova Monteleone)? Please answer yes or no. Yes +01aca2d30b43d05f.jpg Is this a picture of Dorfkirche Gnandstein? Please answer yes or no. No +01b12048bab7439f.jpg Is this a photo of Kaptai Lake? Please answer yes or no. Yes +01b12048bab7439f.jpg Is this a photo of Castle of the Conception? Please answer yes or no. No +01b715ceb7050bce.jpg Is this a picture of Kalamita fortress? Please answer yes or no. Yes +01b715ceb7050bce.jpg Is this a picture of Augsburger Hotelturm? Please answer yes or no. No +01b9b58a573c45f0.jpg Is this a picture of St Mary Redcliffe? Please answer yes or no. Yes +01b9b58a573c45f0.jpg Is this a picture of Kirche zur Heiligsten Dreifaltigkeit, Vienna? Please answer yes or no. No +01bc5557ed2616ad.jpg Is this a photo of Intercession of the Theotokos church in Lutsk? Please answer yes or no. Yes +01bc5557ed2616ad.jpg Is this a photo of Cap 110? Please answer yes or no. No +01c0e0ff84631905.jpg Is this a photo of Ribblehead Viaduct? Please answer yes or no. Yes +01c0e0ff84631905.jpg Is this a photo of Church of Saint George in Koptevo? Please answer yes or no. No +01d6e386c4a01ab3.jpg Is this a picture of Rotes Rathaus? Please answer yes or no. Yes +01d6e386c4a01ab3.jpg Is this a picture of Mahipatgad? Please answer yes or no. No +01d9bef0120557a5.jpg Is this a photo of Andronikov Monastery? Please answer yes or no. Yes +01d9bef0120557a5.jpg Is this a photo of Ghatkopar? Please answer yes or no. No +01e198bea8285816.jpg Is this an image of Saint Basil church, Volodymyr-Volynskyi? Please answer yes or no. Yes +01e198bea8285816.jpg Is this an image of Hoher Markt, Vienna? Please answer yes or no. No +01f23b4c5253e9c6.jpg Is this a picture of Goethes Gartenhaus? Please answer yes or no. Yes +01f23b4c5253e9c6.jpg Is this a picture of Lamba, Shetland Islands? Please answer yes or no. No +01ffc77bdf7fe084.jpg Is this a picture of Oldemorstoft? Please answer yes or no. Yes +01ffc77bdf7fe084.jpg Is this a picture of Roman Catholic Diocese of Antwerp? Please answer yes or no. No +0200cce957e9c4a6.jpg Is this a photo of Malmuerta Tower? Please answer yes or no. Yes +0200cce957e9c4a6.jpg Is this a photo of Monte Lema? Please answer yes or no. No +0202aac6b0bfbcd8.jpg Is this a picture of Gettysburg National Military Park? Please answer yes or no. Yes +0202aac6b0bfbcd8.jpg Is this a picture of Auron? Please answer yes or no. No +0205343362e60daf.jpg Is this a picture of Church of Our Lady of Sorrows in Riga? Please answer yes or no. Yes +0205343362e60daf.jpg Is this a picture of Simon en Judaskerk (Ootmarsum)? Please answer yes or no. No +020ae4dd487a8ab0.jpg Is this a photo of Kashueti Church? Please answer yes or no. Yes +020ae4dd487a8ab0.jpg Is this a photo of Colorado State Capitol? Please answer yes or no. No +02259144c3f094f0.jpg Is this a picture of The American Adventure? Please answer yes or no. Yes +02259144c3f094f0.jpg Is this a picture of Langenwaldschanze? Please answer yes or no. No +0237fc92f6b31aeb.jpg Is this a photo of Abbaye de Mortemer? Please answer yes or no. Yes +0237fc92f6b31aeb.jpg Is this a photo of St Thomas Rest Park? Please answer yes or no. No +0240727dbd2b8531.jpg Is this a photo of Nicolaikerk Appingedam? Please answer yes or no. Yes +0240727dbd2b8531.jpg Is this a photo of Cutimbo? Please answer yes or no. No +02545ca051c6c480.jpg Is this a photo of Podilski Tovtry National Nature Park? Please answer yes or no. Yes +02545ca051c6c480.jpg Is this a photo of Lyndonville, Vermont? Please answer yes or no. No +02598153523880aa.jpg Is this a photo of Marilao Church? Please answer yes or no. Yes +02598153523880aa.jpg Is this a photo of Madliena, Malta? Please answer yes or no. No +02737bb70957827d.jpg Is this an image of White Pagoda in Zhakou? Please answer yes or no. Yes +02737bb70957827d.jpg Is this an image of Norrstrandskyrkan? Please answer yes or no. No +0273f127895337be.jpg Is this a photo of Ringkirche? Please answer yes or no. Yes +0273f127895337be.jpg Is this a photo of Huxi Mosque? Please answer yes or no. No +02785530b6ac4f56.jpg Is this a picture of Church of Nicholas the Wet? Please answer yes or no. Yes +02785530b6ac4f56.jpg Is this a picture of Razula? Please answer yes or no. No +028348883685db69.jpg Is this an image of Saint Andrew the Apostle Church? Please answer yes or no. Yes +028348883685db69.jpg Is this an image of EYE Film Institute Netherlands? Please answer yes or no. No +029431b10472c228.jpg Is this a picture of Santa Caterina (Sassari)? Please answer yes or no. Yes +029431b10472c228.jpg Is this a picture of Former Tainan Assembly Hall? Please answer yes or no. No +0295877d5cd76190.jpg Is this a picture of Groothoofdspoort? Please answer yes or no. Yes +0295877d5cd76190.jpg Is this a picture of 707 17th Street? Please answer yes or no. No +02b16429304b00b7.jpg Is this a picture of De Haag? Please answer yes or no. Yes +02b16429304b00b7.jpg Is this a picture of Villa Melchett? Please answer yes or no. No +02b7f3ae549582f1.jpg Is this a photo of Pont-canal de Digoin? Please answer yes or no. Yes +02b7f3ae549582f1.jpg Is this a photo of Trenton Battle Monument? Please answer yes or no. No +02d11e392d3ab93f.jpg Is this an image of Santa Monica Parish Church in Angat, Bulacan? Please answer yes or no. Yes +02d11e392d3ab93f.jpg Is this an image of Bangkok City Pillar Shrine? Please answer yes or no. No +02e38cd5a3850f60.jpg Is this a photo of Palazzo Cavalli Franchetti (Venice)? Please answer yes or no. Yes +02e38cd5a3850f60.jpg Is this a photo of Isbister Chambered Cairn? Please answer yes or no. No +02e3e418198427e7.jpg Is this an image of Goldener Reiter? Please answer yes or no. Yes +02e3e418198427e7.jpg Is this an image of Palacio de la Mosquera? Please answer yes or no. No +030d9a26b7d5755e.jpg Is this a photo of Medvedgrad? Please answer yes or no. Yes +030d9a26b7d5755e.jpg Is this a photo of Tammisto? Please answer yes or no. No +0317376043fb00eb.jpg Is this a photo of Butler's Wharf? Please answer yes or no. Yes +0317376043fb00eb.jpg Is this a photo of Church of Saints Philip and James (Lelekovice)? Please answer yes or no. No +03280db41fd4e4f5.jpg Is this a picture of Oxford Street? Please answer yes or no. Yes +03280db41fd4e4f5.jpg Is this a picture of Blake Fell? Please answer yes or no. No +034733eb61d663cd.jpg Is this a picture of Colonna dell'Immacolata (Rome)? Please answer yes or no. Yes +034733eb61d663cd.jpg Is this a picture of Brookmill Park? Please answer yes or no. No +034d066523bb20d1.jpg Is this a photo of Ibrahim-al-Ibrahim Mosque? Please answer yes or no. Yes +034d066523bb20d1.jpg Is this a photo of Atami Castle? Please answer yes or no. No +036b60d84c6ebc63.jpg Is this a photo of Frederik V statue in Amalienborg Palace? Please answer yes or no. Yes +036b60d84c6ebc63.jpg Is this a photo of Karl-Bittel-Park? Please answer yes or no. No +03729394d18f7e9a.jpg Is this an image of Sapporo Clock Tower? Please answer yes or no. Yes +03729394d18f7e9a.jpg Is this an image of Saint Nikolaus Church (Beuster)? Please answer yes or no. No +0384fd0664c0ca62.jpg Is this an image of St. Martin (Sontheim)? Please answer yes or no. Yes +0384fd0664c0ca62.jpg Is this an image of Hot Creek (Mono County, California)? Please answer yes or no. No +038bdff81be029a6.jpg Is this an image of Fosso Reale (Livorno)? Please answer yes or no. Yes +038bdff81be029a6.jpg Is this an image of Farlington Marshes? Please answer yes or no. No +039bbdf71cd17b3f.jpg Is this a picture of Qiantang River Bridge? Please answer yes or no. Yes +039bbdf71cd17b3f.jpg Is this a picture of Casa Museo Boschi Di Stefano (Milan)? Please answer yes or no. No +03aa5dd2e3aab548.jpg Is this a photo of Central Police Station? Please answer yes or no. Yes +03aa5dd2e3aab548.jpg Is this a photo of Kontai-ji? Please answer yes or no. No +03c4b1901e4195f4.jpg Is this an image of St. Johanniskirche Eppendorf? Please answer yes or no. Yes +03c4b1901e4195f4.jpg Is this an image of Maidenhead Bridge? Please answer yes or no. No +03c8b2c1306c0038.jpg Is this an image of Ermita de Sant Joan (Montserrat)? Please answer yes or no. Yes +03c8b2c1306c0038.jpg Is this an image of Hamilton Grange National Memorial? Please answer yes or no. No +03d5e3bfc958be38.jpg Is this an image of Pietrarsa railway museum? Please answer yes or no. Yes +03d5e3bfc958be38.jpg Is this an image of Washita Battlefield National Historic Site? Please answer yes or no. No +03dc7df1f1df7b8f.jpg Is this a picture of Yevhen Paton Bridge? Please answer yes or no. Yes +03dc7df1f1df7b8f.jpg Is this a picture of Minoan Villa of Makrygialos? Please answer yes or no. No +03f65f52c0077b6c.jpg Is this a photo of Reading Lighthouse? Please answer yes or no. Yes +03f65f52c0077b6c.jpg Is this a photo of St Nicholas Kirk, Aberdeen? Please answer yes or no. No +03f873280d814ae7.jpg Is this a photo of Clarence Dock (Leeds)? Please answer yes or no. Yes +03f873280d814ae7.jpg Is this a photo of Madonna delle Grazie (Roccastrada)? Please answer yes or no. No +03f8abf9f8473a6f.jpg Is this a picture of Parque nacional del Teide? Please answer yes or no. Yes +03f8abf9f8473a6f.jpg Is this a picture of Church of Saint Mary the Virgin, Skiemonys? Please answer yes or no. No +03fb778d388ad9a0.jpg Is this an image of Mercado Central de Santiago? Please answer yes or no. Yes +03fb778d388ad9a0.jpg Is this an image of Castelo de Alegrete? Please answer yes or no. No +041da60e053c5906.jpg Is this a picture of Schloss Weesenstein? Please answer yes or no. Yes +041da60e053c5906.jpg Is this a picture of Mas Cabrafiga? Please answer yes or no. No +042ca5291cefe0b2.jpg Is this a picture of Pfarrkirche hl. Augustinus, Perchtoldsdorf? Please answer yes or no. Yes +042ca5291cefe0b2.jpg Is this a picture of Mark Twain Riverboat? Please answer yes or no. No +0443da4f4fcdea52.jpg Is this an image of Port of Funchal? Please answer yes or no. Yes +0443da4f4fcdea52.jpg Is this an image of Rauscherpark? Please answer yes or no. No +04531b37dfe5c991.jpg Is this a picture of Roebling Suspension Bridge? Please answer yes or no. Yes +04531b37dfe5c991.jpg Is this a picture of St. James Episcopal Church (Wilmington, North Carolina)? Please answer yes or no. No +04534ba775aa38ad.jpg Is this an image of Cork City Gaol? Please answer yes or no. Yes +04534ba775aa38ad.jpg Is this an image of Taramati Baradari? Please answer yes or no. No +04590c1648dd25a0.jpg Is this an image of Zhejiang Sci-Tech University? Please answer yes or no. Yes +04590c1648dd25a0.jpg Is this an image of Khalti Lake? Please answer yes or no. No +045b65ea196dac3f.jpg Is this a photo of Republican architecture in Manizales? Please answer yes or no. Yes +045b65ea196dac3f.jpg Is this a photo of Kayauchi-banta Cliff? Please answer yes or no. No +045ec5631638d290.jpg Is this a picture of Baldachin of Saint Peter's Basilica? Please answer yes or no. Yes +045ec5631638d290.jpg Is this a picture of Jean Baptiste Point Du Sable Homesite? Please answer yes or no. No +04a5226570dce6c3.jpg Is this a picture of Church of the Assumption of the Virgin Mary in Chrudim? Please answer yes or no. Yes +04a5226570dce6c3.jpg Is this a picture of Queen Elizabeth Country Park? Please answer yes or no. No +04b6c84ce89461ec.jpg Is this an image of Tomb of Xerxes I? Please answer yes or no. Yes +04b6c84ce89461ec.jpg Is this an image of Park HaMesila? Please answer yes or no. No +04d67117549e07a8.jpg Is this a photo of Laguna de Salinas? Please answer yes or no. Yes +04d67117549e07a8.jpg Is this a photo of Raspyatsky Monastery (Serpukhov)? Please answer yes or no. No +04ebe2885134597f.jpg Is this a picture of The South Africa (Delville Wood) National Memorial? Please answer yes or no. Yes +04ebe2885134597f.jpg Is this a picture of Imogiri? Please answer yes or no. No +050464e4ebc0b36d.jpg Is this an image of Tai Wong Ye Temple, Wong Chuk Hang? Please answer yes or no. Yes +050464e4ebc0b36d.jpg Is this an image of Hierve el Agua? Please answer yes or no. No +05095dd2b6715fc3.jpg Is this an image of Sampsonievsky Cathedral? Please answer yes or no. Yes +05095dd2b6715fc3.jpg Is this an image of I. and E. Greenwald Steam Engine No. 1058? Please answer yes or no. No +05685b1070b828f5.jpg Is this a photo of Wurstelprater? Please answer yes or no. Yes +05685b1070b828f5.jpg Is this a photo of Nanmyaebonthar Sannandawya Sandamuni Pagoda? Please answer yes or no. No +05b01b255cec2bfb.jpg Is this a photo of Puthoorppilly Sree Krishnaswamy Temple? Please answer yes or no. Yes +05b01b255cec2bfb.jpg Is this a photo of Villa Reale (Marlia)? Please answer yes or no. No +05bcf05b4460d2c3.jpg Is this an image of Museo archeologico nazionale dell'Umbria? Please answer yes or no. Yes +05bcf05b4460d2c3.jpg Is this an image of Mode Gakuen Cocoon Tower? Please answer yes or no. No +05c3aa404351f407.jpg Is this an image of Pasaje Gutierrez (Valladolid)? Please answer yes or no. Yes +05c3aa404351f407.jpg Is this an image of Pieve di Santa Maria a Carli? Please answer yes or no. No +05d56e249af839e7.jpg Is this a photo of Serbian Orthodox Cathedral in Sarajevo? Please answer yes or no. Yes +05d56e249af839e7.jpg Is this a photo of Seewaldsee (Sankt Koloman)? Please answer yes or no. No +06134984e2aa9bcc.jpg Is this a picture of Grote Kerk (Groede)? Please answer yes or no. Yes +06134984e2aa9bcc.jpg Is this a picture of Goldeck? Please answer yes or no. No +0613d7e8f292ee15.jpg Is this a photo of Museo Archeologico G. Rambotti? Please answer yes or no. Yes +0613d7e8f292ee15.jpg Is this a photo of Sanrei? Please answer yes or no. No +062298815e9197e7.jpg Is this a photo of Liuyang Confucius Temple? Please answer yes or no. Yes +062298815e9197e7.jpg Is this a photo of Astley Castle? Please answer yes or no. No +06483ba2bff074bc.jpg Is this an image of Museo Universitario del Chopo? Please answer yes or no. Yes +06483ba2bff074bc.jpg Is this an image of Cedar Park, Seattle, Washington? Please answer yes or no. No +06740b93ad1f6c02.jpg Is this a picture of Borisoglebsky Monastery (Borisoglebsky)? Please answer yes or no. Yes +06740b93ad1f6c02.jpg Is this a picture of Aletsch Arena? Please answer yes or no. No +067ea27ec08c9354.jpg Is this a picture of Schlosskirche St. Aegidien (Bernburg)? Please answer yes or no. Yes +067ea27ec08c9354.jpg Is this a picture of The Haining? Please answer yes or no. No +06a85d0b5d70881d.jpg Is this an image of Silwan? Please answer yes or no. Yes +06a85d0b5d70881d.jpg Is this an image of Jagdschloss Platte? Please answer yes or no. No +06ab45292ecd8bf7.jpg Is this a picture of Maribor Regional Museum? Please answer yes or no. Yes +06ab45292ecd8bf7.jpg Is this a picture of Myres Castle? Please answer yes or no. No +06b6531dab42ccab.jpg Is this a picture of Roland (Riga)? Please answer yes or no. Yes +06b6531dab42ccab.jpg Is this a picture of Wudangshan Shangdi Temple? Please answer yes or no. No +06d039ff97afef04.jpg Is this a picture of Arch of Galerius (Thessaloniki)? Please answer yes or no. Yes +06d039ff97afef04.jpg Is this a picture of Catholic Church of Wanchin? Please answer yes or no. No +06e453b06a3e4054.jpg Is this an image of Pendeen Lighthouse? Please answer yes or no. Yes +06e453b06a3e4054.jpg Is this an image of Silberbach (Heubach)? Please answer yes or no. No +06e927f784642ce4.jpg Is this a picture of Museo Delta Antico? Please answer yes or no. Yes +06e927f784642ce4.jpg Is this a picture of OSTEC Exhibition Hall? Please answer yes or no. No +070c8d6326a87397.jpg Is this an image of Lanxmeerpoort? Please answer yes or no. Yes +070c8d6326a87397.jpg Is this an image of Tramway du Cap-Ferret? Please answer yes or no. No +0713451824a443d4.jpg Is this a picture of Castillo de Sax? Please answer yes or no. Yes +0713451824a443d4.jpg Is this a picture of Roman Catholic Diocese of Orlando? Please answer yes or no. No +0743e58605497887.jpg Is this an image of Capitoline temple (Brescia)? Please answer yes or no. Yes +0743e58605497887.jpg Is this an image of Burg Horn? Please answer yes or no. No +075264ec41057049.jpg Is this a picture of Imatrankoski? Please answer yes or no. Yes +075264ec41057049.jpg Is this a picture of Jarlsberg travbane? Please answer yes or no. No +07551cf96af77299.jpg Is this a photo of Friedhof Wilmersdorf? Please answer yes or no. Yes +07551cf96af77299.jpg Is this a photo of Partisan cemetery in Mostar? Please answer yes or no. No +075f8d86303a020e.jpg Is this a photo of Church of the Nativity (Bethlehem)? Please answer yes or no. Yes +075f8d86303a020e.jpg Is this a photo of Pennsylvania Railroad Office Building? Please answer yes or no. No +07ac52cf4620519f.jpg Is this a photo of San Pietro (Reggio Emilia)? Please answer yes or no. Yes +07ac52cf4620519f.jpg Is this a photo of Plaza de toros de La Condomina? Please answer yes or no. No +07b04584e67cb39d.jpg Is this a picture of Smithfield, Dublin? Please answer yes or no. Yes +07b04584e67cb39d.jpg Is this a picture of Novyi Svit Sanctuary? Please answer yes or no. No +07ed43dd266b669b.jpg Is this a photo of Rocky Mountain National Park? Please answer yes or no. Yes +07ed43dd266b669b.jpg Is this a photo of Col-des-Roches? Please answer yes or no. No +083dd5a3ba3d96dc.jpg Is this a picture of Teddington Cemetery? Please answer yes or no. Yes +083dd5a3ba3d96dc.jpg Is this a picture of Khutir Nadia? Please answer yes or no. No +08442f24f3c25565.jpg Is this a photo of Abdijkerk Loosduinen? Please answer yes or no. Yes +08442f24f3c25565.jpg Is this a photo of Rinsey Head? Please answer yes or no. No +08a0d2fd3fe6dc95.jpg Is this a picture of Dreifaltigkeitskirche (Ulm)? Please answer yes or no. Yes +08a0d2fd3fe6dc95.jpg Is this a picture of Wat Pa Mamuang? Please answer yes or no. No +08c43cd8f2be6709.jpg Is this a picture of Parchi di Nervi? Please answer yes or no. Yes +08c43cd8f2be6709.jpg Is this a picture of Altlayer Schweiz? Please answer yes or no. No +09232aeaf46dec64.jpg Is this a picture of Canadian Museum of Nature? Please answer yes or no. Yes +09232aeaf46dec64.jpg Is this a picture of Gwangtonggwan? Please answer yes or no. No +092d0859ff6424de.jpg Is this a photo of Kasteel Lunenburg? Please answer yes or no. Yes +092d0859ff6424de.jpg Is this a photo of Thomas Paine Cottage? Please answer yes or no. No +092e075e8e44c228.jpg Is this a picture of ChaoTianGong? Please answer yes or no. Yes +092e075e8e44c228.jpg Is this a picture of Pygmalion Theater Wien? Please answer yes or no. No +093ad5564a004653.jpg Is this a picture of Crescent Lake (Dunhuang)? Please answer yes or no. Yes +093ad5564a004653.jpg Is this a picture of Kirjurinluoto Arena? Please answer yes or no. No +0962855b67159733.jpg Is this a photo of Monastery of Monsalud? Please answer yes or no. Yes +0962855b67159733.jpg Is this a photo of ToonSeum? Please answer yes or no. No +09a51d517f1a72d5.jpg Is this a picture of Ford's Theatre? Please answer yes or no. Yes +09a51d517f1a72d5.jpg Is this a picture of Burg Hachen? Please answer yes or no. No +09ae0628a75e4fcc.jpg Is this a picture of Onze-Lieve-Vrouw Hemelvaartkerk (Zottegem)? Please answer yes or no. Yes +09ae0628a75e4fcc.jpg Is this a picture of Salvation Army in Canada? Please answer yes or no. No +09b7acf87c8046ec.jpg Is this an image of Church of the Epiphany (Yaroslavl)? Please answer yes or no. Yes +09b7acf87c8046ec.jpg Is this an image of Szlenkier Palace? Please answer yes or no. No +09b94a2f304a540c.jpg Is this a photo of Abbaye Notre-Dame du Val? Please answer yes or no. Yes +09b94a2f304a540c.jpg Is this a photo of Hotarumachi, Osaka? Please answer yes or no. No +09ce89cbf237da03.jpg Is this a picture of Church of Saint Mary Magdalene (Brno)? Please answer yes or no. Yes +09ce89cbf237da03.jpg Is this a picture of Wain Wath Force? Please answer yes or no. No +0a3035bfca2ab920.jpg Is this an image of Ortigia? Please answer yes or no. Yes +0a3035bfca2ab920.jpg Is this an image of Montmayeur castle? Please answer yes or no. No +0ab2ed007db301d5.jpg Is this a picture of Highgate Cemetery? Please answer yes or no. Yes +0ab2ed007db301d5.jpg Is this a picture of Isola Maggiore? Please answer yes or no. No +0acc12d1261d99ca.jpg Is this an image of Wat Chedi Si Hong? Please answer yes or no. Yes +0acc12d1261d99ca.jpg Is this an image of Leschi Park? Please answer yes or no. No +0adb088e24d88123.jpg Is this a picture of Iglesia de Santo Domingo (La Serena)? Please answer yes or no. Yes +0adb088e24d88123.jpg Is this a picture of Canelles de Baix (la Vall de Bianya)? Please answer yes or no. No +0ae3355e0a54db21.jpg Is this a photo of Grimspound? Please answer yes or no. Yes +0ae3355e0a54db21.jpg Is this a photo of Lough Owel? Please answer yes or no. No +0b4240b5ef69362e.jpg Is this a photo of Saint Anne church in Nowa Ruda? Please answer yes or no. Yes +0b4240b5ef69362e.jpg Is this a photo of Bray Wick? Please answer yes or no. No +0b64944086932c90.jpg Is this a picture of Alte Pfarrkirche St. Martin (Moosach)? Please answer yes or no. Yes +0b64944086932c90.jpg Is this a picture of Maya Devi Temple, Lumbini? Please answer yes or no. No +0b8fa4152f1bc3ed.jpg Is this a picture of Lac du Salagou? Please answer yes or no. Yes +0b8fa4152f1bc3ed.jpg Is this a picture of Holsljunga kyrka? Please answer yes or no. No +0ba6fa7ade050d49.jpg Is this a photo of Churches in Zemaiciu Kalvarija? Please answer yes or no. Yes +0ba6fa7ade050d49.jpg Is this a photo of Waiola Church? Please answer yes or no. No +0bd72080836eac97.jpg Is this a picture of Castelo de Sesimbra? Please answer yes or no. Yes +0bd72080836eac97.jpg Is this a picture of Gouffre Berger? Please answer yes or no. No +0bdd954254310e73.jpg Is this a photo of Melrose House? Please answer yes or no. Yes +0bdd954254310e73.jpg Is this a photo of Former water tower (Tsubame, Niigata)? Please answer yes or no. No +0c06fe97f38f6f88.jpg Is this a photo of Belvoir Castle? Please answer yes or no. Yes +0c06fe97f38f6f88.jpg Is this a photo of Su'ao Cold Spring? Please answer yes or no. No +0c6b0c5d2a401285.jpg Is this a photo of Pelham Crescent? Please answer yes or no. Yes +0c6b0c5d2a401285.jpg Is this a photo of Manila Ocean Park? Please answer yes or no. No +0cc5201e37961a1d.jpg Is this a photo of Adare Manor? Please answer yes or no. Yes +0cc5201e37961a1d.jpg Is this a photo of Leopoldspark? Please answer yes or no. No +0cd8fda9668b3ed2.jpg Is this a picture of Church of the Holy Mandylion (Klyazma)? Please answer yes or no. Yes +0cd8fda9668b3ed2.jpg Is this a picture of Kalayar Kovil? Please answer yes or no. No +0cf2e28266a0f1a1.jpg Is this an image of Piazza Bra (Verona)? Please answer yes or no. Yes +0cf2e28266a0f1a1.jpg Is this an image of Keeneland? Please answer yes or no. No +0cf48b827c9c99de.jpg Is this a photo of Beatus Rhenanus Bridge? Please answer yes or no. Yes +0cf48b827c9c99de.jpg Is this a photo of Ossian Hall? Please answer yes or no. No +0d23f6c345b68c05.jpg Is this an image of Monte Olivia? Please answer yes or no. Yes +0d23f6c345b68c05.jpg Is this an image of Crackstate? Please answer yes or no. No +0d70bceb2557d1f4.jpg Is this an image of Spaso-Preobrazhensky Cathedral (Pereslavl-Zalessky)? Please answer yes or no. Yes +0d70bceb2557d1f4.jpg Is this an image of Manta Rota? Please answer yes or no. No +0d7716f4aed0c9fa.jpg Is this a photo of Willemsbrug in Rotterdam? Please answer yes or no. Yes +0d7716f4aed0c9fa.jpg Is this a photo of Grumentum, Parco archeologico? Please answer yes or no. No +0d89a8a5e8c269e4.jpg Is this an image of Burg Normannstein? Please answer yes or no. Yes +0d89a8a5e8c269e4.jpg Is this an image of Cape Borda Lighthouse? Please answer yes or no. No +0e04b4f7f9801ea9.jpg Is this a picture of Vase sculpture, Bremen? Please answer yes or no. Yes +0e04b4f7f9801ea9.jpg Is this a picture of Mount Wilhelm? Please answer yes or no. No +0e0511c6e33636ca.jpg Is this a picture of Victoria Stadium? Please answer yes or no. Yes +0e0511c6e33636ca.jpg Is this a picture of Villa Belvedere (San Giuliano Terme)? Please answer yes or no. No +0e553d5d99e21c32.jpg Is this a picture of Mustafa Pasha Bridge? Please answer yes or no. Yes +0e553d5d99e21c32.jpg Is this a picture of Scrattons Eco Park? Please answer yes or no. No +0eb857c972ddc4c8.jpg Is this an image of Nabq Protected Area? Please answer yes or no. Yes +0eb857c972ddc4c8.jpg Is this an image of Maletsunyane Falls? Please answer yes or no. No +0f92862c0366ab60.jpg Is this a photo of Cotehele? Please answer yes or no. Yes +0f92862c0366ab60.jpg Is this a photo of Mozirje Grove? Please answer yes or no. No +0fc48f210ea7d363.jpg Is this a photo of Schauenburg (Oberkirch)? Please answer yes or no. Yes +0fc48f210ea7d363.jpg Is this a photo of Dartford Library? Please answer yes or no. No +100e5402cfb2dfcf.jpg Is this a picture of Nanzenji? Please answer yes or no. Yes +100e5402cfb2dfcf.jpg Is this a picture of Bagakain Lake? Please answer yes or no. No +101786bd2ec5800e.jpg Is this a picture of Church of the Merciful Saviour in Kuskovo (Moscow)? Please answer yes or no. Yes +101786bd2ec5800e.jpg Is this a picture of Central Board of Secondary Education? Please answer yes or no. No +1023b7ec812bb4c6.jpg Is this a photo of St. Joseph's Cathedral, Dunedin? Please answer yes or no. Yes +1023b7ec812bb4c6.jpg Is this a photo of Fagervik Church? Please answer yes or no. No +1053f4cebfceabd8.jpg Is this an image of Woolwich foot tunnel? Please answer yes or no. Yes +1053f4cebfceabd8.jpg Is this an image of Kirchenbibliothek St. Marien Barth? Please answer yes or no. No +10a944858a4283e0.jpg Is this a picture of Harding Tomb? Please answer yes or no. Yes +10a944858a4283e0.jpg Is this a picture of Kartause Ittingen? Please answer yes or no. No +10b2b8741f283a3c.jpg Is this a photo of De Bataaf, Winterswijk? Please answer yes or no. Yes +10b2b8741f283a3c.jpg Is this a photo of Porte Guillaume-Lion? Please answer yes or no. No +11b6bb2c35f4ee22.jpg Is this an image of Ku-ring-gai Chase National Park? Please answer yes or no. Yes +11b6bb2c35f4ee22.jpg Is this an image of Saint Mark's Coptic Orthodox Cathedral, Cairo? Please answer yes or no. No +127be7d34a21e5ed.jpg Is this a picture of Beachy Head? Please answer yes or no. Yes +127be7d34a21e5ed.jpg Is this a picture of Jindo Bridge? Please answer yes or no. No +130846b4f29fe4ba.jpg Is this a photo of Duomo (Viterbo)? Please answer yes or no. Yes +130846b4f29fe4ba.jpg Is this a photo of Raffles Praslin? Please answer yes or no. No +13a563a1a8ef38ae.jpg Is this a photo of Torre del Rellotge d'Olesa de Montserrat? Please answer yes or no. Yes +13a563a1a8ef38ae.jpg Is this a photo of Salzburg Museum? Please answer yes or no. No +14d3aba340d2fba9.jpg Is this an image of Regard Saint-Martin? Please answer yes or no. Yes +14d3aba340d2fba9.jpg Is this an image of Mount Donna Buang? Please answer yes or no. No +15b75eb6fca4aeb6.jpg Is this a photo of City Duma building (Saint Petersburg)? Please answer yes or no. Yes +15b75eb6fca4aeb6.jpg Is this a photo of Komadome-Hachiman-jinja? Please answer yes or no. No +15ec5d2c0898a56e.jpg Is this a photo of Kids Marina Rotterdam? Please answer yes or no. Yes +15ec5d2c0898a56e.jpg Is this a photo of Kirkwood Observatory? Please answer yes or no. No +15f326a030396dba.jpg Is this a picture of Macquarie Centre? Please answer yes or no. Yes +15f326a030396dba.jpg Is this a picture of California Historical Landmarks in San Mateo County, California? Please answer yes or no. No +162a77504f4e686e.jpg Is this an image of Wieliczka Castle? Please answer yes or no. Yes +162a77504f4e686e.jpg Is this an image of Naval Station Guantanamo Bay? Please answer yes or no. No +16b76bdd7eaaef1b.jpg Is this a picture of Evangelische Kirche Weinfelden? Please answer yes or no. Yes +16b76bdd7eaaef1b.jpg Is this a picture of Gorg Blau? Please answer yes or no. No +16d55614b78b6b8b.jpg Is this an image of Church of the Assumption in Bishche? Please answer yes or no. Yes +16d55614b78b6b8b.jpg Is this an image of Water of Girvan? Please answer yes or no. No +17e0cd732e35d4aa.jpg Is this a photo of Millennium Tower (San Francisco, California)? Please answer yes or no. Yes +17e0cd732e35d4aa.jpg Is this a photo of New Orleans Botanical Garden? Please answer yes or no. No +18608ca3b785abf8.jpg Is this a picture of Solvognen? Please answer yes or no. Yes +18608ca3b785abf8.jpg Is this a picture of Schenley Farms National Historic District? Please answer yes or no. No +197e440391d309f8.jpg Is this a picture of Skansin? Please answer yes or no. Yes +197e440391d309f8.jpg Is this a picture of Bardstown Historic District? Please answer yes or no. No +19fe04377265454e.jpg Is this a picture of Alexander Nevsky Church, Belgrade? Please answer yes or no. Yes +19fe04377265454e.jpg Is this a picture of Seppiko-Mineyama Hyogo Prefectural Nature Park? Please answer yes or no. No +1a32959819038291.jpg Is this an image of Id Kah Mosque? Please answer yes or no. Yes +1a32959819038291.jpg Is this an image of Katharinenkirche (Zwickau)? Please answer yes or no. No +1ac0e3c493073995.jpg Is this an image of Church of Saint John the Baptist (Jihlava)? Please answer yes or no. Yes +1ac0e3c493073995.jpg Is this an image of Silte kyrka? Please answer yes or no. No +1d4be36153741972.jpg Is this a picture of Marie Guyart Building? Please answer yes or no. Yes +1d4be36153741972.jpg Is this a picture of Motol? Please answer yes or no. No +1f233b807b660d25.jpg Is this a photo of Aussichtsturm Pyramidenkogel? Please answer yes or no. Yes +1f233b807b660d25.jpg Is this a photo of Vvedenskoe cemetery? Please answer yes or no. No +27a4267b9ea26cd0.jpg Is this an image of Oude Salviuskerk (Limbricht)? Please answer yes or no. Yes +27a4267b9ea26cd0.jpg Is this an image of Festung Heldrungen? Please answer yes or no. No +2a895eb889d6fd99.jpg Is this an image of Beijing Guozijian? Please answer yes or no. Yes +2a895eb889d6fd99.jpg Is this an image of Klinikkirche (Pfafferode)? Please answer yes or no. No +2f25654bcb445452.jpg Is this a photo of Rinku Gate Tower Building? Please answer yes or no. Yes +2f25654bcb445452.jpg Is this a photo of Bigorski Monastery? Please answer yes or no. No +3c89eb4043bcefac.jpg Is this a picture of Begijnhofkerk (Mechelen)? Please answer yes or no. Yes +3c89eb4043bcefac.jpg Is this a picture of Panther Hollow Lake? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/numerical_calculation.txt b/internvl_chat_audio/eval/mme/Your_Results/numerical_calculation.txt new file mode 100644 index 00000000..b85b310e --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/numerical_calculation.txt @@ -0,0 +1,40 @@ +0001.png Is the answer to the arithmetic question in the image 225? Please answer yes or no. Yes +0001.png Is the answer to the arithmetic question in the image 1515? Please answer yes or no. No +0002.png Is the answer to the arithmetic question in the image 340? Please answer yes or no. Yes +0002.png Is the answer to the arithmetic question in the image 17? Please answer yes or no. No +0003.png Is the answer to the arithmetic question in the image 65? Please answer yes or no. Yes +0003.png Is the answer to the arithmetic question in the image 56? Please answer yes or no. No +0004.png Is the answer to the arithmetic question in the image 33? Please answer yes or no. Yes +0004.png Is the answer to the arithmetic question in the image 32? Please answer yes or no. No +0005.png Is the area of the square in the picture equal to 40? Please answer yes or no. Yes +0005.png Is the area of the square in the picture equal to 8? Please answer yes or no. No +0006.png Is the area of the square in the picture equal to 9? Please answer yes or no. Yes +0006.png Is the area of the square in the picture equal to 3? Please answer yes or no. No +0007.png Is the answer to the arithmetic question in the image 49? Please answer yes or no. Yes +0007.png Is the answer to the arithmetic question in the image 39? Please answer yes or no. No +0008.png Should the value of "a" in the picture equal 7? Please answer yes or no. Yes +0008.png Should the value of "a" in the picture equal 14? Please answer yes or no. No +0009.png Should the value of "a" in the picture equal 2? Please answer yes or no. Yes +0009.png Should the value of "a" in the picture equal 3? Please answer yes or no. No +0010.png Is the answer to the arithmetic question in the image 13? Please answer yes or no. Yes +0010.png Is the answer to the arithmetic question in the image 12? Please answer yes or no. No +0011.png Is the area of the parallelogram in the picture equal to 24? Please answer yes or no. Yes +0011.png Is the area of the parallelogram in the picture equal to 6? Please answer yes or no. No +0012.png Should the value of "a" in the picture equal 9? Please answer yes or no. Yes +0012.png Should the value of "a" in the picture equal 1? Please answer yes or no. No +0013.png Is the area of the right triangle in the picture equal to 24? Please answer yes or no. Yes +0013.png Is the area of the right triangle in the picture equal to 8? Please answer yes or no. No +0014.png Is the answer to the arithmetic question in the image 200? Please answer yes or no. Yes +0014.png Is the answer to the arithmetic question in the image 400? Please answer yes or no. No +0015.png Is the answer to the arithmetic question in the image 11? Please answer yes or no. Yes +0015.png Is the answer to the arithmetic question in the image 111? Please answer yes or no. No +0016.png Is the answer to the arithmetic question in the image 9? Please answer yes or no. Yes +0016.png Is the answer to the arithmetic question in the image 16? Please answer yes or no. No +0017.png Is the answer to the arithmetic question in the image 14? Please answer yes or no. Yes +0017.png Is the answer to the arithmetic question in the image 83? Please answer yes or no. No +0018.png Should the value of "a" in the picture equal 3? Please answer yes or no. Yes +0018.png Should the value of "a" in the picture equal 2? Please answer yes or no. No +0019.png Is the answer to the arithmetic question in the image 18? Please answer yes or no. Yes +0019.png Is the answer to the arithmetic question in the image 36? Please answer yes or no. No +0020.png Is the answer to the arithmetic question in the image 9? Please answer yes or no. Yes +0020.png Is the answer to the arithmetic question in the image 45? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/position.txt b/internvl_chat_audio/eval/mme/Your_Results/position.txt new file mode 100644 index 00000000..62111f71 --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/position.txt @@ -0,0 +1,60 @@ +000000006471.jpg Is the cricket bat above the batter's body? Please answer yes or no. Yes +000000006471.jpg Is the cricket bat under the batter's body Please answer yes or no. No +000000007281.jpg Is the sea behind people in the image? Please answer yes or no. Yes +000000007281.jpg Is the sea in front of people in the image? Please answer yes or no. No +000000014038.jpg Is the refrigerator on the left side of the picture? Please answer yes or no. Yes +000000014038.jpg Is the refrigerator on the right side of the picture Please answer yes or no. No +000000031248.jpg Is there a sofa in the middle of potted plants in the image? Please answer yes or no. Yes +000000031248.jpg Is there a sofa in the right side of potted plants in the image? Please answer yes or no. No +000000048504.jpg Is the gray elephant in front of the brown elephant? Please answer yes or no. Yes +000000048504.jpg Is the brown elephant in front of the gray elephant? Please answer yes or no. No +000000052007.jpg Are the pedestrians on the right of the bus? Please answer yes or no. Yes +000000052007.jpg Are the pedestrians on the left of the bus? Please answer yes or no. No +000000056127.jpg Is the light above the fire hydrant in the image? Please answer yes or no. Yes +000000056127.jpg Is the light under the fire hydrant in the image? Please answer yes or no. No +000000062025.jpg Is the trash can under the cup in the image？ Please answer yes or no. Yes +000000062025.jpg Is the trash can above the cup in the image？ Please answer yes or no. No +000000062808.jpg Is the phone above the pizza in the image? Please answer yes or no. Yes +000000062808.jpg Is the phone under the pizza in the image? Please answer yes or no. No +000000067213.jpg Is the dog above the pool in the image? Please answer yes or no. Yes +000000067213.jpg Is the dog under the pool in the image? Please answer yes or no. No +000000097994.jpg Is the light above the computer in the image? Please answer yes or no. Yes +000000097994.jpg Is the light under the computer in the image? Please answer yes or no. No +000000204871.jpg Is the car on the right side of the fire hydrant in the picture? Please answer yes or no. Yes +000000204871.jpg Is the car on the left side of the fire hydrant in the picture? Please answer yes or no. No +000000206487.jpg Is the motorcycle on the right side of the bus? Please answer yes or no. Yes +000000206487.jpg Is the motorcycle on the left side of the bus Please answer yes or no. No +000000211825.jpg Is the cake on the left side of the camera? Please answer yes or no. Yes +000000211825.jpg Is the cake on the right side of the camera? Please answer yes or no. No +000000212800.jpg Is the blue umbrella under the black umbrella? Please answer yes or no. Yes +000000212800.jpg Is the blue umbrella above the black umbrella? Please answer yes or no. No +000000395701.jpg Is the TV on the left of the bookshelf? Please answer yes or no. Yes +000000395701.jpg Is the TV on the right of the bookshelf? Please answer yes or no. No +000000395801.jpg Is the clock above people? Please answer yes or no. Yes +000000395801.jpg Is the clock under people? Please answer yes or no. No +000000405970.jpg Is the grey sofa on the right of the TV? Please answer yes or no. Yes +000000405970.jpg Is the grey sofa on the left of the TV? Please answer yes or no. No +000000426241.jpg Is the white mouse on the right of the black keyboard? Please answer yes or no. Yes +000000426241.jpg Is the white mouse on the left of the black keyboard? Please answer yes or no. No +000000450303.jpg Is the monitor on top of a person? Please answer yes or no. Yes +000000450303.jpg Is the monitor under the person? Please answer yes or no. No +000000458410.jpg Is the TV on the left of the lamp? Please answer yes or no. Yes +000000458410.jpg Is the TV on the right of the lamp? Please answer yes or no. No +000000472046.jpg Is the pineapple on the left of the pot in the image? Please answer yes or no. Yes +000000472046.jpg Is the pineapple on the right of the pot in the image? Please answer yes or no. No +000000477955.jpg Is the person under the kite? Please answer yes or no. Yes +000000477955.jpg Is the person above the kite? Please answer yes or no. No +000000482585.jpg Is the person on the right of the train? Please answer yes or no. Yes +000000482585.jpg Is the person on the left of the train? Please answer yes or no. No +000000494869.jpg Is the baby on the right of the dog in the image? Please answer yes or no. Yes +000000494869.jpg Is the baby on the left of the dog in the image? Please answer yes or no. No +000000509699.jpg Is the mirror above the TV? Please answer yes or no. Yes +000000509699.jpg Is the mirror under the TV? Please answer yes or no. No +000000519569.jpg Is the vase on the left of the bottle? Please answer yes or no. Yes +000000519569.jpg Is the vase on the right of the bottle? Please answer yes or no. No +000000530162.jpg Is the big red and black umbrella on the top of people? Please answer yes or no. Yes +000000530162.jpg Is the big red and black umbrella under people? Please answer yes or no. No +000000551660.jpg Is the spoon in the bowl? Please answer yes or no. Yes +000000551660.jpg Is the spoon out of the bowl? Please answer yes or no. No +000000578922.jpg Is the vase on the left of the toothbrush? Please answer yes or no. Yes +000000578922.jpg Is the vase on the right of the toothbrush? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/posters.txt b/internvl_chat_audio/eval/mme/Your_Results/posters.txt new file mode 100644 index 00000000..4919bd80 --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/posters.txt @@ -0,0 +1,294 @@ +tt0048028.jpg Is this movie directed by elia kazan? Please answer yes or no. Yes +tt0048028.jpg Is this movie directed by maneesh sharma? Please answer yes or no. No +tt0053221.jpg Is this movie titled rio bravo (1959)? Please answer yes or no. Yes +tt0053221.jpg Is this movie titled the matrix (1999)? Please answer yes or no. No +tt0062622.jpg Is this movie originated from the country or region of uk? Please answer yes or no. Yes +tt0062622.jpg Is this movie originated from the country or region of china? Please answer yes or no. No +tt0063442.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0063442.jpg Is this movie originated from the country or region of japan? Please answer yes or no. No +tt0065724.jpg Is this movie directed by bob rafelson? Please answer yes or no. Yes +tt0065724.jpg Is this movie directed by anthony minghella? Please answer yes or no. No +tt0067116.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0067116.jpg Is this movie originated from the country or region of uk? Please answer yes or no. No +tt0067140.jpg Is this movie titled duck, you sucker (1971)? Please answer yes or no. Yes +tt0067140.jpg Is this movie titled crimson tide (1995)? Please answer yes or no. No +tt0067185.jpg Is this movie directed by hal ashby? Please answer yes or no. Yes +tt0067185.jpg Is this movie directed by john hillcoat? Please answer yes or no. No +tt0068646.jpg Is this movie titled the godfather (1972)? Please answer yes or no. Yes +tt0068646.jpg Is this movie titled bring on the melody (2017)? Please answer yes or no. No +tt0070291.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0070291.jpg Is this movie originated from the country or region of spain? Please answer yes or no. No +tt0071315.jpg Is this movie titled chinatown (1974)? Please answer yes or no. Yes +tt0071315.jpg Is this movie titled planet of the apes (1968)? Please answer yes or no. No +tt0074119.jpg Is this movie directed by alan j. pakula? Please answer yes or no. Yes +tt0074119.jpg Is this movie directed by stanley kubrick? Please answer yes or no. No +tt0074749.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0074749.jpg Is this movie originated from the country or region of venezuela? Please answer yes or no. No +tt0082186.jpg Is this movie directed by desmond davis? Please answer yes or no. Yes +tt0082186.jpg Is this movie directed by edward zwick? Please answer yes or no. No +tt0083907.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0083907.jpg Is this movie originated from the country or region of bulgaria? Please answer yes or no. No +tt0083946.jpg Is this movie originated from the country or region of west germany? Please answer yes or no. Yes +tt0083946.jpg Is this movie originated from the country or region of denmark? Please answer yes or no. No +tt0084787.jpg Is this movie titled the thing (1982)? Please answer yes or no. Yes +tt0084787.jpg Is this movie titled jay and silent bob strike back (2001)? Please answer yes or no. No +tt0086250.jpg Is this movie directed by brian de palma? Please answer yes or no. Yes +tt0086250.jpg Is this movie directed by patrice leconte? Please answer yes or no. No +tt0093565.jpg Is this movie titled moonstruck (1987)? Please answer yes or no. Yes +tt0093565.jpg Is this movie titled now you see me (2013)? Please answer yes or no. No +tt0093773.jpg Is this movie titled predator (1987)? Please answer yes or no. Yes +tt0093773.jpg Is this movie titled the cell (2000)? Please answer yes or no. No +tt0094291.jpg Is this movie directed by oliver stone? Please answer yes or no. Yes +tt0094291.jpg Is this movie directed by raja gosnell? Please answer yes or no. No +tt0096446.jpg Is this movie directed by ron howard? Please answer yes or no. Yes +tt0096446.jpg Is this movie directed by peter jackson? Please answer yes or no. No +tt0096463.jpg Is this movie directed by mike nichols? Please answer yes or no. Yes +tt0096463.jpg Is this movie directed by cary joji fukunaga? Please answer yes or no. No +tt0096895.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0096895.jpg Is this movie originated from the country or region of china? Please answer yes or no. No +tt0097576.jpg Is this movie directed by steven spielberg? Please answer yes or no. Yes +tt0097576.jpg Is this movie directed by francis ford coppola? Please answer yes or no. No +tt0098635.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0098635.jpg Is this movie originated from the country or region of spain? Please answer yes or no. No +tt0098724.jpg Is this movie titled sex, lies, and videotape (1989)? Please answer yes or no. Yes +tt0098724.jpg Is this movie titled final destination (2000)? Please answer yes or no. No +tt0099653.jpg Is this movie directed by jerry zucker? Please answer yes or no. Yes +tt0099653.jpg Is this movie directed by felix chong,alan mak? Please answer yes or no. No +tt0100112.jpg Is this movie originated from the country or region of france? Please answer yes or no. Yes +tt0100112.jpg Is this movie originated from the country or region of new zealand? Please answer yes or no. No +tt0100150.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0100150.jpg Is this movie originated from the country or region of hong kong? Please answer yes or no. No +tt0100935.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0100935.jpg Is this movie originated from the country or region of canada? Please answer yes or no. No +tt0101889.jpg Is this movie titled the fisher king (1991)? Please answer yes or no. Yes +tt0101889.jpg Is this movie titled a good day to die hard (2013)? Please answer yes or no. No +tt0101921.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0101921.jpg Is this movie originated from the country or region of australia? Please answer yes or no. No +tt0107653.jpg Is this movie titled naked (1993)? Please answer yes or no. Yes +tt0107653.jpg Is this movie titled captain america: civil war (2016)? Please answer yes or no. No +tt0107808.jpg Is this movie titled a perfect world (1993)? Please answer yes or no. Yes +tt0107808.jpg Is this movie titled harold and maude (1971)? Please answer yes or no. No +tt0110201.jpg Is this movie titled hail the judge (1994)? Please answer yes or no. Yes +tt0110201.jpg Is this movie titled batman (1989)? Please answer yes or no. No +tt0112740.jpg Is this movie titled crimson tide (1995)? Please answer yes or no. Yes +tt0112740.jpg Is this movie titled insidious: the last key (2018)? Please answer yes or no. No +tt0113497.jpg Is this movie titled jumanji (1995)? Please answer yes or no. Yes +tt0113497.jpg Is this movie titled morgan (2016)? Please answer yes or no. No +tt0114369.jpg Is this movie titled se7en (1995)? Please answer yes or no. Yes +tt0114369.jpg Is this movie titled a perfect world (1993)? Please answer yes or no. No +tt0114388.jpg Is this movie directed by ang lee? Please answer yes or no. Yes +tt0114388.jpg Is this movie directed by david lowery? Please answer yes or no. No +tt0116209.jpg Is this movie directed by anthony minghella? Please answer yes or no. Yes +tt0116209.jpg Is this movie directed by john cassavetes? Please answer yes or no. No +tt0116629.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0116629.jpg Is this movie originated from the country or region of australia? Please answer yes or no. No +tt0118636.jpg Is this movie titled apt pupil (1998)? Please answer yes or no. Yes +tt0118636.jpg Is this movie titled a good day to die hard (2013)? Please answer yes or no. No +tt0118655.jpg Is this movie directed by jay roach? Please answer yes or no. Yes +tt0118655.jpg Is this movie directed by peter jackson? Please answer yes or no. No +tt0118715.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0118715.jpg Is this movie originated from the country or region of venezuela? Please answer yes or no. No +tt0119303.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0119303.jpg Is this movie originated from the country or region of spain? Please answer yes or no. No +tt0120738.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0120738.jpg Is this movie originated from the country or region of germany? Please answer yes or no. No +tt0129387.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0129387.jpg Is this movie originated from the country or region of germany? Please answer yes or no. No +tt0133093.jpg Is this movie directed by lana wachowski,lilly wachowski? Please answer yes or no. Yes +tt0133093.jpg Is this movie directed by siu-tung ching? Please answer yes or no. No +tt0163025.jpg Is this movie directed by joe johnston? Please answer yes or no. Yes +tt0163025.jpg Is this movie directed by adam robitel? Please answer yes or no. No +tt0166924.jpg Is this movie titled mulholland drive (2001)? Please answer yes or no. Yes +tt0166924.jpg Is this movie titled lucky number slevin (2006)? Please answer yes or no. No +tt0167261.jpg Is this movie directed by peter jackson? Please answer yes or no. Yes +tt0167261.jpg Is this movie directed by gabe ibáñez? Please answer yes or no. No +tt0182789.jpg Is this movie titled bicentennial man (1999)? Please answer yes or no. Yes +tt0182789.jpg Is this movie titled ocean's eleven (2001)? Please answer yes or no. No +tt0187078.jpg Is this movie titled gone in 60 seconds (2000)? Please answer yes or no. Yes +tt0187078.jpg Is this movie titled avatar (2009)? Please answer yes or no. No +tt0195714.jpg Is this movie directed by james wong? Please answer yes or no. Yes +tt0195714.jpg Is this movie directed by bob rafelson? Please answer yes or no. No +tt0209958.jpg Is this movie titled the cell (2000)? Please answer yes or no. Yes +tt0209958.jpg Is this movie titled the matrix revolutions (2003)? Please answer yes or no. No +tt0232500.jpg Is this movie titled the fast and the furious (2001)? Please answer yes or no. Yes +tt0232500.jpg Is this movie titled east of eden (1955)? Please answer yes or no. No +tt0240772.jpg Is this movie directed by steven soderbergh? Please answer yes or no. Yes +tt0240772.jpg Is this movie directed by paul mcguigan? Please answer yes or no. No +tt0242653.jpg Is this movie titled the matrix revolutions (2003)? Please answer yes or no. Yes +tt0242653.jpg Is this movie titled sense and sensibility (1995)? Please answer yes or no. No +tt0244244.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0244244.jpg Is this movie originated from the country or region of argentina? Please answer yes or no. No +tt0258463.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0258463.jpg Is this movie originated from the country or region of italy? Please answer yes or no. No +tt0261392.jpg Is this movie titled jay and silent bob strike back (2001)? Please answer yes or no. Yes +tt0261392.jpg Is this movie titled avatar (2009)? Please answer yes or no. No +tt0264395.jpg Is this movie originated from the country or region of germany? Please answer yes or no. Yes +tt0264395.jpg Is this movie originated from the country or region of australia? Please answer yes or no. No +tt0266697.jpg Is this movie directed by quentin tarantino? Please answer yes or no. Yes +tt0266697.jpg Is this movie directed by peyton reed? Please answer yes or no. No +tt0268978.jpg Is this movie titled a beautiful mind (2001)? Please answer yes or no. Yes +tt0268978.jpg Is this movie titled blue jasmine (2013)? Please answer yes or no. No +tt0318627.jpg Is this movie titled resident evil: apocalypse (2004)? Please answer yes or no. Yes +tt0318627.jpg Is this movie titled the meddler (2015)? Please answer yes or no. No +tt0328107.jpg Is this movie titled man on fire (2004)? Please answer yes or no. Yes +tt0328107.jpg Is this movie titled real steel (2011)? Please answer yes or no. No +tt0338751.jpg Is this movie directed by martin scorsese? Please answer yes or no. Yes +tt0338751.jpg Is this movie directed by chris columbus? Please answer yes or no. No +tt0341495.jpg Is this movie directed by siu-tung ching? Please answer yes or no. Yes +tt0341495.jpg Is this movie directed by lorenzo vigas? Please answer yes or no. No +tt0351977.jpg Is this movie directed by kevin bray? Please answer yes or no. Yes +tt0351977.jpg Is this movie directed by joe johnston? Please answer yes or no. No +tt0369702.jpg Is this movie originated from the country or region of spain? Please answer yes or no. Yes +tt0369702.jpg Is this movie originated from the country or region of france? Please answer yes or no. No +tt0373469.jpg Is this movie directed by shane black? Please answer yes or no. Yes +tt0373469.jpg Is this movie directed by dennis dugan? Please answer yes or no. No +tt0378194.jpg Is this movie directed by quentin tarantino? Please answer yes or no. Yes +tt0378194.jpg Is this movie directed by todd phillips? Please answer yes or no. No +tt0379786.jpg Is this movie titled serenity (2005)? Please answer yes or no. Yes +tt0379786.jpg Is this movie titled bad genius (2017)? Please answer yes or no. No +tt0393109.jpg Is this movie titled brick (2005)? Please answer yes or no. Yes +tt0393109.jpg Is this movie titled sense and sensibility (1995)? Please answer yes or no. No +tt0399201.jpg Is this movie directed by michael bay? Please answer yes or no. Yes +tt0399201.jpg Is this movie directed by angelina jolie? Please answer yes or no. No +tt0425210.jpg Is this movie titled lucky number slevin (2006)? Please answer yes or no. Yes +tt0425210.jpg Is this movie titled the godfather (1972)? Please answer yes or no. No +tt0433035.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0433035.jpg Is this movie originated from the country or region of new zealand? Please answer yes or no. No +tt0443680.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0443680.jpg Is this movie originated from the country or region of germany? Please answer yes or no. No +tt0449088.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0449088.jpg Is this movie originated from the country or region of japan? Please answer yes or no. No +tt0450259.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt0450259.jpg Is this movie originated from the country or region of china? Please answer yes or no. No +tt0462499.jpg Is this movie titled rambo (2008)? Please answer yes or no. Yes +tt0462499.jpg Is this movie titled walking tall (2004)? Please answer yes or no. No +tt0499549.jpg Is this movie directed by james cameron? Please answer yes or no. Yes +tt0499549.jpg Is this movie directed by jerry zucker? Please answer yes or no. No +tt0780653.jpg Is this movie titled the wolfman (2010)? Please answer yes or no. Yes +tt0780653.jpg Is this movie titled wild at heart (1990)? Please answer yes or no. No +tt0829482.jpg Is this movie directed by greg mottola? Please answer yes or no. Yes +tt0829482.jpg Is this movie directed by jing wong? Please answer yes or no. No +tt0848228.jpg Is this movie titled the avengers (2012)? Please answer yes or no. Yes +tt0848228.jpg Is this movie titled wolf warrior 2 (2017)? Please answer yes or no. No +tt0898367.jpg Is this movie directed by john hillcoat? Please answer yes or no. Yes +tt0898367.jpg Is this movie directed by franklin j. schaffner? Please answer yes or no. No +tt0942385.jpg Is this movie titled tropic thunder (2008)? Please answer yes or no. Yes +tt0942385.jpg Is this movie titled bad teacher (2011)? Please answer yes or no. No +tt0993846.jpg Is this movie titled the wolf of wall street (2013)? Please answer yes or no. Yes +tt0993846.jpg Is this movie titled lucky number slevin (2006)? Please answer yes or no. No +tt1017460.jpg Is this movie directed by vincenzo natali? Please answer yes or no. Yes +tt1017460.jpg Is this movie directed by raja gosnell? Please answer yes or no. No +tt1057500.jpg Is this movie directed by clint eastwood? Please answer yes or no. Yes +tt1057500.jpg Is this movie directed by john cassavetes? Please answer yes or no. No +tt1068680.jpg Is this movie titled yes man (2008)? Please answer yes or no. Yes +tt1068680.jpg Is this movie titled rio bravo (1959)? Please answer yes or no. No +tt1099212.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt1099212.jpg Is this movie originated from the country or region of japan? Please answer yes or no. No +tt1119646.jpg Is this movie directed by todd phillips? Please answer yes or no. Yes +tt1119646.jpg Is this movie directed by franklin j. schaffner? Please answer yes or no. No +tt1124037.jpg Is this movie directed by gary ross? Please answer yes or no. Yes +tt1124037.jpg Is this movie directed by francis ford coppola? Please answer yes or no. No +tt1229822.jpg Is this movie titled jane eyre (2011)? Please answer yes or no. Yes +tt1229822.jpg Is this movie titled kiss kiss bang bang (2005)? Please answer yes or no. No +tt1284575.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt1284575.jpg Is this movie originated from the country or region of bulgaria? Please answer yes or no. No +tt1291150.jpg Is this movie directed by jonathan liebesman? Please answer yes or no. Yes +tt1291150.jpg Is this movie directed by masahide ichii? Please answer yes or no. No +tt1300851.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt1300851.jpg Is this movie originated from the country or region of germany? Please answer yes or no. No +tt1305806.jpg Is this movie titled the secret in their eyes (2009)? Please answer yes or no. Yes +tt1305806.jpg Is this movie titled ocean's eleven (2001)? Please answer yes or no. No +tt1343092.jpg Is this movie directed by baz luhrmann? Please answer yes or no. Yes +tt1343092.jpg Is this movie directed by lorenzo vigas? Please answer yes or no. No +tt1385826.jpg Is this movie titled the adjustment bureau (2011)? Please answer yes or no. Yes +tt1385826.jpg Is this movie titled the last of sheila (1973)? Please answer yes or no. No +tt1403865.jpg Is this movie directed by ethan coen,joel coen? Please answer yes or no. Yes +tt1403865.jpg Is this movie directed by john hillcoat? Please answer yes or no. No +tt1438176.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt1438176.jpg Is this movie originated from the country or region of west germany? Please answer yes or no. No +tt1454029.jpg Is this movie titled the help (2011)? Please answer yes or no. Yes +tt1454029.jpg Is this movie titled avatar (2009)? Please answer yes or no. No +tt1560747.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt1560747.jpg Is this movie originated from the country or region of canada? Please answer yes or no. No +tt1564367.jpg Is this movie titled just go with it (2011)? Please answer yes or no. Yes +tt1564367.jpg Is this movie titled journey to the west (2013)? Please answer yes or no. No +tt1606378.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt1606378.jpg Is this movie originated from the country or region of uk? Please answer yes or no. No +tt1637725.jpg Is this movie titled ted (2012)? Please answer yes or no. Yes +tt1637725.jpg Is this movie titled lost in space (1998)? Please answer yes or no. No +tt1670345.jpg Is this movie titled now you see me (2013)? Please answer yes or no. Yes +tt1670345.jpg Is this movie titled bicentennial man (1999)? Please answer yes or no. No +tt1703957.jpg Is this movie titled genius (2016)? Please answer yes or no. Yes +tt1703957.jpg Is this movie titled fan (2016)? Please answer yes or no. No +tt1722484.jpg Is this movie directed by hermine huntgeburth? Please answer yes or no. Yes +tt1722484.jpg Is this movie directed by todd phillips? Please answer yes or no. No +tt1800241.jpg Is this movie directed by david o. russell? Please answer yes or no. Yes +tt1800241.jpg Is this movie directed by raja gosnell? Please answer yes or no. No +tt1809398.jpg Is this movie titled unbroken (2014)? Please answer yes or no. Yes +tt1809398.jpg Is this movie titled home alone 3 (1997)? Please answer yes or no. No +tt1971325.jpg Is this movie originated from the country or region of bulgaria? Please answer yes or no. Yes +tt1971325.jpg Is this movie originated from the country or region of spain? Please answer yes or no. No +tt1974419.jpg Is this movie titled the neon demon (2016)? Please answer yes or no. Yes +tt1974419.jpg Is this movie titled man on fire (2004)? Please answer yes or no. No +tt2017561.jpg Is this movie directed by stephen chow,chi-kin kwok? Please answer yes or no. Yes +tt2017561.jpg Is this movie directed by craig gillespie? Please answer yes or no. No +tt2078768.jpg Is this movie originated from the country or region of hong kong? Please answer yes or no. Yes +tt2078768.jpg Is this movie originated from the country or region of venezuela? Please answer yes or no. No +tt2132285.jpg Is this movie titled the bling ring (2013)? Please answer yes or no. Yes +tt2132285.jpg Is this movie titled tropic thunder (2008)? Please answer yes or no. No +tt2238032.jpg Is this movie titled skiptrace (2016)? Please answer yes or no. Yes +tt2238032.jpg Is this movie titled the bourne identity (2002)? Please answer yes or no. No +tt2334873.jpg Is this movie titled blue jasmine (2013)? Please answer yes or no. Yes +tt2334873.jpg Is this movie titled a good day to die hard (2013)? Please answer yes or no. No +tt2788732.jpg Is this movie titled pete's dragon (2016)? Please answer yes or no. Yes +tt2788732.jpg Is this movie titled willow (1988)? Please answer yes or no. No +tt2802144.jpg Is this movie directed by matthew vaughn? Please answer yes or no. Yes +tt2802144.jpg Is this movie directed by dietrich brüggemann? Please answer yes or no. No +tt2908856.jpg Is this movie directed by israel horovitz? Please answer yes or no. Yes +tt2908856.jpg Is this movie directed by mike nichols? Please answer yes or no. No +tt2980516.jpg Is this movie titled the theory of everything (2014)? Please answer yes or no. Yes +tt2980516.jpg Is this movie titled stations of the cross (2014)? Please answer yes or no. No +tt3077214.jpg Is this movie titled suffragette (2015)? Please answer yes or no. Yes +tt3077214.jpg Is this movie titled east of eden (1955)? Please answer yes or no. No +tt3316960.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt3316960.jpg Is this movie originated from the country or region of west germany? Please answer yes or no. No +tt3465916.jpg Is this movie originated from the country or region of germany? Please answer yes or no. Yes +tt3465916.jpg Is this movie originated from the country or region of spain? Please answer yes or no. No +tt3495026.jpg Is this movie titled fan (2016)? Please answer yes or no. Yes +tt3495026.jpg Is this movie titled from afar (2015)? Please answer yes or no. No +tt3498820.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt3498820.jpg Is this movie originated from the country or region of germany? Please answer yes or no. No +tt3700804.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt3700804.jpg Is this movie originated from the country or region of uk? Please answer yes or no. No +tt3824458.jpg Is this movie titled tangerine (2015)? Please answer yes or no. Yes +tt3824458.jpg Is this movie titled miller's crossing (1990)? Please answer yes or no. No +tt3860916.jpg Is this movie directed by ben howling? Please answer yes or no. Yes +tt3860916.jpg Is this movie directed by quentin tarantino? Please answer yes or no. No +tt4046784.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt4046784.jpg Is this movie originated from the country or region of uk? Please answer yes or no. No +tt4242158.jpg Is this movie titled lost in hong kong (2015)? Please answer yes or no. Yes +tt4242158.jpg Is this movie titled harold and maude (1971)? Please answer yes or no. No +tt4273292.jpg Is this movie directed by babak anvari? Please answer yes or no. Yes +tt4273292.jpg Is this movie directed by ethan coen,joel coen? Please answer yes or no. No +tt4501454.jpg Is this movie directed by lorene scafaria? Please answer yes or no. Yes +tt4501454.jpg Is this movie directed by herbert ross? Please answer yes or no. No +tt4520364.jpg Is this movie directed by luke scott? Please answer yes or no. Yes +tt4520364.jpg Is this movie directed by dominic sena? Please answer yes or no. No +tt4651520.jpg Is this movie originated from the country or region of usa? Please answer yes or no. Yes +tt4651520.jpg Is this movie originated from the country or region of china? Please answer yes or no. No +tt4721400.jpg Is this movie directed by lorenzo vigas? Please answer yes or no. Yes +tt4721400.jpg Is this movie directed by jonathan liebesman? Please answer yes or no. No +tt4972062.jpg Is this movie directed by mike birbiglia? Please answer yes or no. Yes +tt4972062.jpg Is this movie directed by david fincher? Please answer yes or no. No +tt5564148.jpg Is this movie directed by masahide ichii? Please answer yes or no. Yes +tt5564148.jpg Is this movie directed by dietrich brüggemann? Please answer yes or no. No +tt5688868.jpg Is this movie titled primal rage: the legend of konga (2018)? Please answer yes or no. Yes +tt5688868.jpg Is this movie titled the neon demon (2016)? Please answer yes or no. No +tt5726086.jpg Is this movie directed by adam robitel? Please answer yes or no. Yes +tt5726086.jpg Is this movie directed by gore verbinski? Please answer yes or no. No +tt6788942.jpg Is this movie originated from the country or region of thailand? Please answer yes or no. Yes +tt6788942.jpg Is this movie originated from the country or region of argentina? Please answer yes or no. No +tt7055592.jpg Is this movie titled brotherhood of blades ii: the infernal battlefield (2017)? Please answer yes or no. Yes +tt7055592.jpg Is this movie titled stations of the cross (2014)? Please answer yes or no. No +tt7131870.jpg Is this movie directed by jing wu? Please answer yes or no. Yes +tt7131870.jpg Is this movie directed by david lynch? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/scene.txt b/internvl_chat_audio/eval/mme/Your_Results/scene.txt new file mode 100644 index 00000000..0a29e410 --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/scene.txt @@ -0,0 +1,400 @@ +Places365_val_00000001.jpg Is this picture captured in a place of greenhouse indoor? Please answer yes or no. Yes +Places365_val_00000001.jpg Is this picture captured in a place of waiting room? Please answer yes or no. No +Places365_val_00000002.jpg Is this photo taken in a place of wet bar? Please answer yes or no. Yes +Places365_val_00000002.jpg Is this photo taken in a place of stage indoor? Please answer yes or no. No +Places365_val_00000003.jpg Is this picture captured in a place of clean room? Please answer yes or no. Yes +Places365_val_00000003.jpg Is this picture captured in a place of bookstore? Please answer yes or no. No +Places365_val_00000004.jpg Is this picture captured in a place of golf course? Please answer yes or no. Yes +Places365_val_00000004.jpg Is this picture captured in a place of computer room? Please answer yes or no. No +Places365_val_00000005.jpg Is this picture captured in a place of rock arch? Please answer yes or no. Yes +Places365_val_00000005.jpg Is this picture captured in a place of desert sand? Please answer yes or no. No +Places365_val_00000006.jpg Is this photo taken in a place of corridor? Please answer yes or no. Yes +Places365_val_00000006.jpg Is this photo taken in a place of jail cell? Please answer yes or no. No +Places365_val_00000007.jpg Does this image describe a place of canyon? Please answer yes or no. Yes +Places365_val_00000007.jpg Does this image describe a place of basement? Please answer yes or no. No +Places365_val_00000008.jpg Is this picture captured in a place of dining room? Please answer yes or no. Yes +Places365_val_00000008.jpg Is this picture captured in a place of ball pit? Please answer yes or no. No +Places365_val_00000009.jpg Is this picture captured in a place of forest broadleaf? Please answer yes or no. Yes +Places365_val_00000009.jpg Is this picture captured in a place of swamp? Please answer yes or no. No +Places365_val_00000010.jpg Does this image describe a place of shopping mall indoor? Please answer yes or no. Yes +Places365_val_00000010.jpg Does this image describe a place of mosque outdoor? Please answer yes or no. No +Places365_val_00000011.jpg Is this photo taken in a place of baseball field? Please answer yes or no. Yes +Places365_val_00000011.jpg Is this photo taken in a place of bus station indoor? Please answer yes or no. No +Places365_val_00000012.jpg Does this image describe a place of campus? Please answer yes or no. Yes +Places365_val_00000012.jpg Does this image describe a place of sandbox? Please answer yes or no. No +Places365_val_00000013.jpg Is this picture captured in a place of beach house? Please answer yes or no. Yes +Places365_val_00000013.jpg Is this picture captured in a place of alcove? Please answer yes or no. No +Places365_val_00000014.jpg Does this image describe a place of art gallery? Please answer yes or no. Yes +Places365_val_00000014.jpg Does this image describe a place of volleyball court outdoor? Please answer yes or no. No +Places365_val_00000015.jpg Does this image describe a place of bus interior? Please answer yes or no. Yes +Places365_val_00000015.jpg Does this image describe a place of butchers shop? Please answer yes or no. No +Places365_val_00000016.jpg Does this image describe a place of gymnasium indoor? Please answer yes or no. Yes +Places365_val_00000016.jpg Does this image describe a place of crevasse? Please answer yes or no. No +Places365_val_00000017.jpg Is this picture captured in a place of glacier? Please answer yes or no. Yes +Places365_val_00000017.jpg Is this picture captured in a place of fishpond? Please answer yes or no. No +Places365_val_00000018.jpg Is this photo taken in a place of nursing home? Please answer yes or no. Yes +Places365_val_00000018.jpg Is this photo taken in a place of server room? Please answer yes or no. No +Places365_val_00000019.jpg Does this image describe a place of storage room? Please answer yes or no. Yes +Places365_val_00000019.jpg Does this image describe a place of aquarium? Please answer yes or no. No +Places365_val_00000020.jpg Is this picture captured in a place of florist shop indoor? Please answer yes or no. Yes +Places365_val_00000020.jpg Is this picture captured in a place of office? Please answer yes or no. No +Places365_val_00000021.jpg Is this picture captured in a place of restaurant kitchen? Please answer yes or no. Yes +Places365_val_00000021.jpg Is this picture captured in a place of garage outdoor? Please answer yes or no. No +Places365_val_00000022.jpg Is this photo taken in a place of recreation room? Please answer yes or no. Yes +Places365_val_00000022.jpg Is this photo taken in a place of runway? Please answer yes or no. No +Places365_val_00000023.jpg Does this image describe a place of schoolhouse? Please answer yes or no. Yes +Places365_val_00000023.jpg Does this image describe a place of raft? Please answer yes or no. No +Places365_val_00000024.jpg Does this image describe a place of amusement arcade? Please answer yes or no. Yes +Places365_val_00000024.jpg Does this image describe a place of medina? Please answer yes or no. No +Places365_val_00000025.jpg Is this photo taken in a place of ocean? Please answer yes or no. Yes +Places365_val_00000025.jpg Is this photo taken in a place of sauna? Please answer yes or no. No +Places365_val_00000026.jpg Does this image describe a place of river? Please answer yes or no. Yes +Places365_val_00000026.jpg Does this image describe a place of corral? Please answer yes or no. No +Places365_val_00000027.jpg Is this photo taken in a place of pub indoor? Please answer yes or no. Yes +Places365_val_00000027.jpg Is this photo taken in a place of basement? Please answer yes or no. No +Places365_val_00000028.jpg Is this picture captured in a place of lecture room? Please answer yes or no. Yes +Places365_val_00000028.jpg Is this picture captured in a place of aqueduct? Please answer yes or no. No +Places365_val_00000029.jpg Is this photo taken in a place of japanese garden? Please answer yes or no. Yes +Places365_val_00000029.jpg Is this photo taken in a place of pharmacy? Please answer yes or no. No +Places365_val_00000030.jpg Does this image describe a place of motel? Please answer yes or no. Yes +Places365_val_00000030.jpg Does this image describe a place of toyshop? Please answer yes or no. No +Places365_val_00000031.jpg Does this image describe a place of assembly line? Please answer yes or no. Yes +Places365_val_00000031.jpg Does this image describe a place of beer garden? Please answer yes or no. No +Places365_val_00000032.jpg Does this image describe a place of campsite? Please answer yes or no. Yes +Places365_val_00000032.jpg Does this image describe a place of elevator shaft? Please answer yes or no. No +Places365_val_00000033.jpg Is this photo taken in a place of florist shop indoor? Please answer yes or no. Yes +Places365_val_00000033.jpg Is this photo taken in a place of sushi bar? Please answer yes or no. No +Places365_val_00000034.jpg Does this image describe a place of racecourse? Please answer yes or no. Yes +Places365_val_00000034.jpg Does this image describe a place of galley? Please answer yes or no. No +Places365_val_00000035.jpg Is this photo taken in a place of kennel outdoor? Please answer yes or no. Yes +Places365_val_00000035.jpg Is this photo taken in a place of bathroom? Please answer yes or no. No +Places365_val_00000036.jpg Is this picture captured in a place of bus interior? Please answer yes or no. Yes +Places365_val_00000036.jpg Is this picture captured in a place of mansion? Please answer yes or no. No +Places365_val_00000037.jpg Does this image describe a place of dressing room? Please answer yes or no. Yes +Places365_val_00000037.jpg Does this image describe a place of chemistry lab? Please answer yes or no. No +Places365_val_00000038.jpg Does this image describe a place of wind farm? Please answer yes or no. Yes +Places365_val_00000038.jpg Does this image describe a place of volleyball court outdoor? Please answer yes or no. No +Places365_val_00000039.jpg Is this picture captured in a place of auto factory? Please answer yes or no. Yes +Places365_val_00000039.jpg Is this picture captured in a place of tree farm? Please answer yes or no. No +Places365_val_00000040.jpg Is this picture captured in a place of beach? Please answer yes or no. Yes +Places365_val_00000040.jpg Is this picture captured in a place of library outdoor? Please answer yes or no. No +Places365_val_00000041.jpg Is this photo taken in a place of mountain? Please answer yes or no. Yes +Places365_val_00000041.jpg Is this photo taken in a place of formal garden? Please answer yes or no. No +Places365_val_00000042.jpg Does this image describe a place of pavilion? Please answer yes or no. Yes +Places365_val_00000042.jpg Does this image describe a place of stable? Please answer yes or no. No +Places365_val_00000043.jpg Does this image describe a place of tree house? Please answer yes or no. Yes +Places365_val_00000043.jpg Does this image describe a place of patio? Please answer yes or no. No +Places365_val_00000044.jpg Is this photo taken in a place of coffee shop? Please answer yes or no. Yes +Places365_val_00000044.jpg Is this photo taken in a place of elevator lobby? Please answer yes or no. No +Places365_val_00000045.jpg Is this photo taken in a place of skyscraper? Please answer yes or no. Yes +Places365_val_00000045.jpg Is this photo taken in a place of forest broadleaf? Please answer yes or no. No +Places365_val_00000046.jpg Does this image describe a place of street? Please answer yes or no. Yes +Places365_val_00000046.jpg Does this image describe a place of golf course? Please answer yes or no. No +Places365_val_00000047.jpg Is this picture captured in a place of alley? Please answer yes or no. Yes +Places365_val_00000047.jpg Is this picture captured in a place of boardwalk? Please answer yes or no. No +Places365_val_00000048.jpg Is this photo taken in a place of supermarket? Please answer yes or no. Yes +Places365_val_00000048.jpg Is this photo taken in a place of creek? Please answer yes or no. No +Places365_val_00000049.jpg Is this picture captured in a place of arena hockey? Please answer yes or no. Yes +Places365_val_00000049.jpg Is this picture captured in a place of ski slope? Please answer yes or no. No +Places365_val_00000050.jpg Is this picture captured in a place of zen garden? Please answer yes or no. Yes +Places365_val_00000050.jpg Is this picture captured in a place of bazaar indoor? Please answer yes or no. No +Places365_val_00000051.jpg Is this picture captured in a place of library indoor? Please answer yes or no. Yes +Places365_val_00000051.jpg Is this picture captured in a place of orchard? Please answer yes or no. No +Places365_val_00000052.jpg Is this photo taken in a place of ice skating rink indoor? Please answer yes or no. Yes +Places365_val_00000052.jpg Is this photo taken in a place of wind farm? Please answer yes or no. No +Places365_val_00000053.jpg Is this picture captured in a place of arena hockey? Please answer yes or no. Yes +Places365_val_00000053.jpg Is this picture captured in a place of loading dock? Please answer yes or no. No +Places365_val_00000054.jpg Does this image describe a place of chalet? Please answer yes or no. Yes +Places365_val_00000054.jpg Does this image describe a place of tower? Please answer yes or no. No +Places365_val_00000055.jpg Does this image describe a place of bedchamber? Please answer yes or no. Yes +Places365_val_00000055.jpg Does this image describe a place of music studio? Please answer yes or no. No +Places365_val_00000056.jpg Is this photo taken in a place of botanical garden? Please answer yes or no. Yes +Places365_val_00000056.jpg Is this photo taken in a place of legislative chamber? Please answer yes or no. No +Places365_val_00000057.jpg Is this picture captured in a place of galley? Please answer yes or no. Yes +Places365_val_00000057.jpg Is this picture captured in a place of physics laboratory? Please answer yes or no. No +Places365_val_00000058.jpg Does this image describe a place of department store? Please answer yes or no. Yes +Places365_val_00000058.jpg Does this image describe a place of rope bridge? Please answer yes or no. No +Places365_val_00000059.jpg Is this photo taken in a place of schoolhouse? Please answer yes or no. Yes +Places365_val_00000059.jpg Is this photo taken in a place of hangar outdoor? Please answer yes or no. No +Places365_val_00000060.jpg Is this photo taken in a place of atrium public? Please answer yes or no. Yes +Places365_val_00000060.jpg Is this photo taken in a place of subway station platform? Please answer yes or no. No +Places365_val_00000061.jpg Is this photo taken in a place of dam? Please answer yes or no. Yes +Places365_val_00000061.jpg Is this photo taken in a place of ice skating rink indoor? Please answer yes or no. No +Places365_val_00000062.jpg Does this image describe a place of army base? Please answer yes or no. Yes +Places365_val_00000062.jpg Does this image describe a place of train station platform? Please answer yes or no. No +Places365_val_00000063.jpg Does this image describe a place of gas station? Please answer yes or no. Yes +Places365_val_00000063.jpg Does this image describe a place of doorway outdoor? Please answer yes or no. No +Places365_val_00000064.jpg Does this image describe a place of ballroom? Please answer yes or no. Yes +Places365_val_00000064.jpg Does this image describe a place of corn field? Please answer yes or no. No +Places365_val_00000065.jpg Does this image describe a place of biology laboratory? Please answer yes or no. Yes +Places365_val_00000065.jpg Does this image describe a place of candy store? Please answer yes or no. No +Places365_val_00000066.jpg Does this image describe a place of pantry? Please answer yes or no. Yes +Places365_val_00000066.jpg Does this image describe a place of lighthouse? Please answer yes or no. No +Places365_val_00000067.jpg Is this picture captured in a place of greenhouse indoor? Please answer yes or no. Yes +Places365_val_00000067.jpg Is this picture captured in a place of pasture? Please answer yes or no. No +Places365_val_00000068.jpg Is this photo taken in a place of kasbah? Please answer yes or no. Yes +Places365_val_00000068.jpg Is this photo taken in a place of swimming hole? Please answer yes or no. No +Places365_val_00000069.jpg Is this picture captured in a place of mausoleum? Please answer yes or no. Yes +Places365_val_00000069.jpg Is this picture captured in a place of alcove? Please answer yes or no. No +Places365_val_00000070.jpg Is this picture captured in a place of booth indoor? Please answer yes or no. Yes +Places365_val_00000070.jpg Is this picture captured in a place of bazaar indoor? Please answer yes or no. No +Places365_val_00000071.jpg Is this picture captured in a place of youth hostel? Please answer yes or no. Yes +Places365_val_00000071.jpg Is this picture captured in a place of landfill? Please answer yes or no. No +Places365_val_00000072.jpg Does this image describe a place of fire escape? Please answer yes or no. Yes +Places365_val_00000072.jpg Does this image describe a place of playroom? Please answer yes or no. No +Places365_val_00000073.jpg Is this photo taken in a place of rice paddy? Please answer yes or no. Yes +Places365_val_00000073.jpg Is this photo taken in a place of bus station indoor? Please answer yes or no. No +Places365_val_00000074.jpg Is this photo taken in a place of mausoleum? Please answer yes or no. Yes +Places365_val_00000074.jpg Is this photo taken in a place of train station platform? Please answer yes or no. No +Places365_val_00000075.jpg Is this photo taken in a place of entrance hall? Please answer yes or no. Yes +Places365_val_00000075.jpg Is this photo taken in a place of residential neighborhood? Please answer yes or no. No +Places365_val_00000076.jpg Is this photo taken in a place of bakery shop? Please answer yes or no. Yes +Places365_val_00000076.jpg Is this photo taken in a place of doorway outdoor? Please answer yes or no. No +Places365_val_00000077.jpg Does this image describe a place of escalator indoor? Please answer yes or no. Yes +Places365_val_00000077.jpg Does this image describe a place of ice shelf? Please answer yes or no. No +Places365_val_00000078.jpg Is this picture captured in a place of elevator shaft? Please answer yes or no. Yes +Places365_val_00000078.jpg Is this picture captured in a place of fire escape? Please answer yes or no. No +Places365_val_00000079.jpg Is this picture captured in a place of industrial area? Please answer yes or no. Yes +Places365_val_00000079.jpg Is this picture captured in a place of botanical garden? Please answer yes or no. No +Places365_val_00000080.jpg Is this photo taken in a place of escalator indoor? Please answer yes or no. Yes +Places365_val_00000080.jpg Is this photo taken in a place of coffee shop? Please answer yes or no. No +Places365_val_00000081.jpg Does this image describe a place of operating room? Please answer yes or no. Yes +Places365_val_00000081.jpg Does this image describe a place of house? Please answer yes or no. No +Places365_val_00000082.jpg Is this photo taken in a place of mosque outdoor? Please answer yes or no. Yes +Places365_val_00000082.jpg Is this photo taken in a place of recreation room? Please answer yes or no. No +Places365_val_00000083.jpg Is this photo taken in a place of cottage? Please answer yes or no. Yes +Places365_val_00000083.jpg Is this photo taken in a place of assembly line? Please answer yes or no. No +Places365_val_00000084.jpg Does this image describe a place of courtyard? Please answer yes or no. Yes +Places365_val_00000084.jpg Does this image describe a place of greenhouse outdoor? Please answer yes or no. No +Places365_val_00000085.jpg Is this photo taken in a place of flea market indoor? Please answer yes or no. Yes +Places365_val_00000085.jpg Is this photo taken in a place of pavilion? Please answer yes or no. No +Places365_val_00000086.jpg Does this image describe a place of banquet hall? Please answer yes or no. Yes +Places365_val_00000086.jpg Does this image describe a place of storage room? Please answer yes or no. No +Places365_val_00000087.jpg Is this photo taken in a place of auto factory? Please answer yes or no. Yes +Places365_val_00000087.jpg Is this photo taken in a place of motel? Please answer yes or no. No +Places365_val_00000088.jpg Is this photo taken in a place of cockpit? Please answer yes or no. Yes +Places365_val_00000088.jpg Is this photo taken in a place of gift shop? Please answer yes or no. No +Places365_val_00000089.jpg Is this photo taken in a place of martial arts gym? Please answer yes or no. Yes +Places365_val_00000089.jpg Is this photo taken in a place of atrium public? Please answer yes or no. No +Places365_val_00000090.jpg Is this picture captured in a place of general store outdoor? Please answer yes or no. Yes +Places365_val_00000090.jpg Is this picture captured in a place of bazaar outdoor? Please answer yes or no. No +Places365_val_00000091.jpg Is this photo taken in a place of motel? Please answer yes or no. Yes +Places365_val_00000091.jpg Is this photo taken in a place of chemistry lab? Please answer yes or no. No +Places365_val_00000092.jpg Is this photo taken in a place of playground? Please answer yes or no. Yes +Places365_val_00000092.jpg Is this photo taken in a place of snowfield? Please answer yes or no. No +Places365_val_00000093.jpg Is this picture captured in a place of basement? Please answer yes or no. Yes +Places365_val_00000093.jpg Is this picture captured in a place of jewelry shop? Please answer yes or no. No +Places365_val_00000094.jpg Is this picture captured in a place of playroom? Please answer yes or no. Yes +Places365_val_00000094.jpg Is this picture captured in a place of pier? Please answer yes or no. No +Places365_val_00000095.jpg Is this photo taken in a place of fabric store? Please answer yes or no. Yes +Places365_val_00000095.jpg Is this photo taken in a place of beauty salon? Please answer yes or no. No +Places365_val_00000096.jpg Is this picture captured in a place of forest path? Please answer yes or no. Yes +Places365_val_00000096.jpg Is this picture captured in a place of balcony interior? Please answer yes or no. No +Places365_val_00000097.jpg Does this image describe a place of hangar indoor? Please answer yes or no. Yes +Places365_val_00000097.jpg Does this image describe a place of vegetable garden? Please answer yes or no. No +Places365_val_00000098.jpg Does this image describe a place of crosswalk? Please answer yes or no. Yes +Places365_val_00000098.jpg Does this image describe a place of snowfield? Please answer yes or no. No +Places365_val_00000099.jpg Does this image describe a place of physics laboratory? Please answer yes or no. Yes +Places365_val_00000099.jpg Does this image describe a place of stage indoor? Please answer yes or no. No +Places365_val_00000100.jpg Is this picture captured in a place of pub indoor? Please answer yes or no. Yes +Places365_val_00000100.jpg Is this picture captured in a place of pier? Please answer yes or no. No +Places365_val_00000101.jpg Is this photo taken in a place of schoolhouse? Please answer yes or no. Yes +Places365_val_00000101.jpg Is this photo taken in a place of vineyard? Please answer yes or no. No +Places365_val_00000102.jpg Is this photo taken in a place of industrial area? Please answer yes or no. Yes +Places365_val_00000102.jpg Is this photo taken in a place of street? Please answer yes or no. No +Places365_val_00000103.jpg Is this picture captured in a place of baseball field? Please answer yes or no. Yes +Places365_val_00000103.jpg Is this picture captured in a place of airfield? Please answer yes or no. No +Places365_val_00000104.jpg Is this picture captured in a place of ice floe? Please answer yes or no. Yes +Places365_val_00000104.jpg Is this picture captured in a place of kindergarden classroom? Please answer yes or no. No +Places365_val_00000105.jpg Is this picture captured in a place of train interior? Please answer yes or no. Yes +Places365_val_00000105.jpg Is this picture captured in a place of movie theater indoor? Please answer yes or no. No +Places365_val_00000106.jpg Is this picture captured in a place of desert sand? Please answer yes or no. Yes +Places365_val_00000106.jpg Is this picture captured in a place of burial chamber? Please answer yes or no. No +Places365_val_00000107.jpg Is this photo taken in a place of castle? Please answer yes or no. Yes +Places365_val_00000107.jpg Is this photo taken in a place of computer room? Please answer yes or no. No +Places365_val_00000108.jpg Does this image describe a place of burial chamber? Please answer yes or no. Yes +Places365_val_00000108.jpg Does this image describe a place of oilrig? Please answer yes or no. No +Places365_val_00000109.jpg Is this photo taken in a place of nursing home? Please answer yes or no. Yes +Places365_val_00000109.jpg Is this photo taken in a place of amusement park? Please answer yes or no. No +Places365_val_00000110.jpg Does this image describe a place of church indoor? Please answer yes or no. Yes +Places365_val_00000110.jpg Does this image describe a place of general store outdoor? Please answer yes or no. No +Places365_val_00000111.jpg Is this picture captured in a place of palace? Please answer yes or no. Yes +Places365_val_00000111.jpg Is this picture captured in a place of tree house? Please answer yes or no. No +Places365_val_00000112.jpg Does this image describe a place of bus station indoor? Please answer yes or no. Yes +Places365_val_00000112.jpg Does this image describe a place of hardware store? Please answer yes or no. No +Places365_val_00000113.jpg Is this picture captured in a place of atrium public? Please answer yes or no. Yes +Places365_val_00000113.jpg Is this picture captured in a place of music studio? Please answer yes or no. No +Places365_val_00000114.jpg Is this photo taken in a place of butte? Please answer yes or no. Yes +Places365_val_00000114.jpg Is this photo taken in a place of valley? Please answer yes or no. No +Places365_val_00000115.jpg Is this picture captured in a place of zen garden? Please answer yes or no. Yes +Places365_val_00000115.jpg Is this picture captured in a place of banquet hall? Please answer yes or no. No +Places365_val_00000116.jpg Is this picture captured in a place of racecourse? Please answer yes or no. Yes +Places365_val_00000116.jpg Is this picture captured in a place of basketball court indoor? Please answer yes or no. No +Places365_val_00000117.jpg Is this picture captured in a place of cliff? Please answer yes or no. Yes +Places365_val_00000117.jpg Is this picture captured in a place of sauna? Please answer yes or no. No +Places365_val_00000118.jpg Is this photo taken in a place of laundromat? Please answer yes or no. Yes +Places365_val_00000118.jpg Is this photo taken in a place of archive? Please answer yes or no. No +Places365_val_00000119.jpg Does this image describe a place of wave? Please answer yes or no. Yes +Places365_val_00000119.jpg Does this image describe a place of shed? Please answer yes or no. No +Places365_val_00000120.jpg Is this photo taken in a place of legislative chamber? Please answer yes or no. Yes +Places365_val_00000120.jpg Is this photo taken in a place of nursery? Please answer yes or no. No +Places365_val_00000121.jpg Is this photo taken in a place of elevator shaft? Please answer yes or no. Yes +Places365_val_00000121.jpg Is this photo taken in a place of berth? Please answer yes or no. No +Places365_val_00000122.jpg Is this photo taken in a place of hayfield? Please answer yes or no. Yes +Places365_val_00000122.jpg Is this photo taken in a place of skyscraper? Please answer yes or no. No +Places365_val_00000123.jpg Is this photo taken in a place of hospital room? Please answer yes or no. Yes +Places365_val_00000123.jpg Is this photo taken in a place of mosque outdoor? Please answer yes or no. No +Places365_val_00000124.jpg Does this image describe a place of swimming hole? Please answer yes or no. Yes +Places365_val_00000124.jpg Does this image describe a place of television room? Please answer yes or no. No +Places365_val_00000125.jpg Is this picture captured in a place of biology laboratory? Please answer yes or no. Yes +Places365_val_00000125.jpg Is this picture captured in a place of lake natural? Please answer yes or no. No +Places365_val_00000126.jpg Is this picture captured in a place of church indoor? Please answer yes or no. Yes +Places365_val_00000126.jpg Is this picture captured in a place of server room? Please answer yes or no. No +Places365_val_00000127.jpg Is this picture captured in a place of temple asia? Please answer yes or no. Yes +Places365_val_00000127.jpg Is this picture captured in a place of ball pit? Please answer yes or no. No +Places365_val_00000128.jpg Is this photo taken in a place of landfill? Please answer yes or no. Yes +Places365_val_00000128.jpg Is this photo taken in a place of viaduct? Please answer yes or no. No +Places365_val_00000129.jpg Does this image describe a place of arena hockey? Please answer yes or no. Yes +Places365_val_00000129.jpg Does this image describe a place of clothing store? Please answer yes or no. No +Places365_val_00000130.jpg Is this photo taken in a place of baseball field? Please answer yes or no. Yes +Places365_val_00000130.jpg Is this photo taken in a place of art gallery? Please answer yes or no. No +Places365_val_00000131.jpg Is this picture captured in a place of gazebo exterior? Please answer yes or no. Yes +Places365_val_00000131.jpg Is this picture captured in a place of driveway? Please answer yes or no. No +Places365_val_00000132.jpg Does this image describe a place of greenhouse outdoor? Please answer yes or no. Yes +Places365_val_00000132.jpg Does this image describe a place of stage indoor? Please answer yes or no. No +Places365_val_00000133.jpg Is this picture captured in a place of beauty salon? Please answer yes or no. Yes +Places365_val_00000133.jpg Is this picture captured in a place of junkyard? Please answer yes or no. No +Places365_val_00000134.jpg Does this image describe a place of ice shelf? Please answer yes or no. Yes +Places365_val_00000134.jpg Does this image describe a place of vegetable garden? Please answer yes or no. No +Places365_val_00000135.jpg Is this picture captured in a place of islet? Please answer yes or no. Yes +Places365_val_00000135.jpg Is this picture captured in a place of recreation room? Please answer yes or no. No +Places365_val_00000136.jpg Is this picture captured in a place of railroad track? Please answer yes or no. Yes +Places365_val_00000136.jpg Is this picture captured in a place of department store? Please answer yes or no. No +Places365_val_00000137.jpg Is this photo taken in a place of bazaar outdoor? Please answer yes or no. Yes +Places365_val_00000137.jpg Is this photo taken in a place of office? Please answer yes or no. No +Places365_val_00000138.jpg Is this photo taken in a place of basement? Please answer yes or no. Yes +Places365_val_00000138.jpg Is this photo taken in a place of banquet hall? Please answer yes or no. No +Places365_val_00000139.jpg Is this picture captured in a place of television studio? Please answer yes or no. Yes +Places365_val_00000139.jpg Is this picture captured in a place of general store indoor? Please answer yes or no. No +Places365_val_00000140.jpg Is this picture captured in a place of mezzanine? Please answer yes or no. Yes +Places365_val_00000140.jpg Is this picture captured in a place of bedroom? Please answer yes or no. No +Places365_val_00000141.jpg Does this image describe a place of roof garden? Please answer yes or no. Yes +Places365_val_00000141.jpg Does this image describe a place of television room? Please answer yes or no. No +Places365_val_00000142.jpg Does this image describe a place of restaurant kitchen? Please answer yes or no. Yes +Places365_val_00000142.jpg Does this image describe a place of fabric store? Please answer yes or no. No +Places365_val_00000143.jpg Is this photo taken in a place of house? Please answer yes or no. Yes +Places365_val_00000143.jpg Is this photo taken in a place of swimming hole? Please answer yes or no. No +Places365_val_00000144.jpg Is this picture captured in a place of vineyard? Please answer yes or no. Yes +Places365_val_00000144.jpg Is this picture captured in a place of castle? Please answer yes or no. No +Places365_val_00000145.jpg Is this picture captured in a place of driveway? Please answer yes or no. Yes +Places365_val_00000145.jpg Is this picture captured in a place of badlands? Please answer yes or no. No +Places365_val_00000146.jpg Does this image describe a place of chemistry lab? Please answer yes or no. Yes +Places365_val_00000146.jpg Does this image describe a place of industrial area? Please answer yes or no. No +Places365_val_00000147.jpg Does this image describe a place of cabin outdoor? Please answer yes or no. Yes +Places365_val_00000147.jpg Does this image describe a place of creek? Please answer yes or no. No +Places365_val_00000148.jpg Is this photo taken in a place of restaurant? Please answer yes or no. Yes +Places365_val_00000148.jpg Is this photo taken in a place of apartment building outdoor? Please answer yes or no. No +Places365_val_00000149.jpg Is this picture captured in a place of auditorium? Please answer yes or no. Yes +Places365_val_00000149.jpg Is this picture captured in a place of rock arch? Please answer yes or no. No +Places365_val_00000150.jpg Is this photo taken in a place of street? Please answer yes or no. Yes +Places365_val_00000150.jpg Is this photo taken in a place of toyshop? Please answer yes or no. No +Places365_val_00000151.jpg Does this image describe a place of lagoon? Please answer yes or no. Yes +Places365_val_00000151.jpg Does this image describe a place of aquarium? Please answer yes or no. No +Places365_val_00000152.jpg Is this picture captured in a place of hangar indoor? Please answer yes or no. Yes +Places365_val_00000152.jpg Is this picture captured in a place of street? Please answer yes or no. No +Places365_val_00000153.jpg Does this image describe a place of bar? Please answer yes or no. Yes +Places365_val_00000153.jpg Does this image describe a place of tree house? Please answer yes or no. No +Places365_val_00000154.jpg Is this picture captured in a place of classroom? Please answer yes or no. Yes +Places365_val_00000154.jpg Is this picture captured in a place of wheat field? Please answer yes or no. No +Places365_val_00000155.jpg Does this image describe a place of creek? Please answer yes or no. Yes +Places365_val_00000155.jpg Does this image describe a place of youth hostel? Please answer yes or no. No +Places365_val_00000156.jpg Does this image describe a place of stable? Please answer yes or no. Yes +Places365_val_00000156.jpg Does this image describe a place of junkyard? Please answer yes or no. No +Places365_val_00000157.jpg Is this picture captured in a place of swamp? Please answer yes or no. Yes +Places365_val_00000157.jpg Is this picture captured in a place of village? Please answer yes or no. No +Places365_val_00000158.jpg Is this picture captured in a place of ice shelf? Please answer yes or no. Yes +Places365_val_00000158.jpg Is this picture captured in a place of motel? Please answer yes or no. No +Places365_val_00000159.jpg Is this picture captured in a place of train interior? Please answer yes or no. Yes +Places365_val_00000159.jpg Is this picture captured in a place of jewelry shop? Please answer yes or no. No +Places365_val_00000160.jpg Does this image describe a place of windmill? Please answer yes or no. Yes +Places365_val_00000160.jpg Does this image describe a place of lake natural? Please answer yes or no. No +Places365_val_00000161.jpg Is this photo taken in a place of archive? Please answer yes or no. Yes +Places365_val_00000161.jpg Is this photo taken in a place of village? Please answer yes or no. No +Places365_val_00000162.jpg Does this image describe a place of train station platform? Please answer yes or no. Yes +Places365_val_00000162.jpg Does this image describe a place of manufactured home? Please answer yes or no. No +Places365_val_00000163.jpg Is this photo taken in a place of chalet? Please answer yes or no. Yes +Places365_val_00000163.jpg Is this photo taken in a place of throne room? Please answer yes or no. No +Places365_val_00000164.jpg Does this image describe a place of lake natural? Please answer yes or no. Yes +Places365_val_00000164.jpg Does this image describe a place of athletic field outdoor? Please answer yes or no. No +Places365_val_00000165.jpg Is this picture captured in a place of playground? Please answer yes or no. Yes +Places365_val_00000165.jpg Is this picture captured in a place of viaduct? Please answer yes or no. No +Places365_val_00000166.jpg Is this photo taken in a place of dorm room? Please answer yes or no. Yes +Places365_val_00000166.jpg Is this photo taken in a place of ski resort? Please answer yes or no. No +Places365_val_00000167.jpg Is this picture captured in a place of flea market indoor? Please answer yes or no. Yes +Places365_val_00000167.jpg Is this picture captured in a place of cliff? Please answer yes or no. No +Places365_val_00000168.jpg Does this image describe a place of airplane cabin? Please answer yes or no. Yes +Places365_val_00000168.jpg Does this image describe a place of rock arch? Please answer yes or no. No +Places365_val_00000169.jpg Is this picture captured in a place of drugstore? Please answer yes or no. Yes +Places365_val_00000169.jpg Is this picture captured in a place of canyon? Please answer yes or no. No +Places365_val_00000170.jpg Is this picture captured in a place of pharmacy? Please answer yes or no. Yes +Places365_val_00000170.jpg Is this picture captured in a place of orchestra pit? Please answer yes or no. No +Places365_val_00000171.jpg Is this picture captured in a place of greenhouse outdoor? Please answer yes or no. Yes +Places365_val_00000171.jpg Is this picture captured in a place of archaelogical excavation? Please answer yes or no. No +Places365_val_00000172.jpg Is this photo taken in a place of bathroom? Please answer yes or no. Yes +Places365_val_00000172.jpg Is this photo taken in a place of auto factory? Please answer yes or no. No +Places365_val_00000173.jpg Is this photo taken in a place of shoe shop? Please answer yes or no. Yes +Places365_val_00000173.jpg Is this photo taken in a place of car interior? Please answer yes or no. No +Places365_val_00000174.jpg Does this image describe a place of moat water? Please answer yes or no. Yes +Places365_val_00000174.jpg Does this image describe a place of marsh? Please answer yes or no. No +Places365_val_00000175.jpg Does this image describe a place of vegetable garden? Please answer yes or no. Yes +Places365_val_00000175.jpg Does this image describe a place of parking garage indoor? Please answer yes or no. No +Places365_val_00000176.jpg Is this picture captured in a place of bowling alley? Please answer yes or no. Yes +Places365_val_00000176.jpg Is this picture captured in a place of hotel outdoor? Please answer yes or no. No +Places365_val_00000177.jpg Does this image describe a place of embassy? Please answer yes or no. Yes +Places365_val_00000177.jpg Does this image describe a place of chemistry lab? Please answer yes or no. No +Places365_val_00000178.jpg Is this photo taken in a place of youth hostel? Please answer yes or no. Yes +Places365_val_00000178.jpg Is this photo taken in a place of hangar indoor? Please answer yes or no. No +Places365_val_00000179.jpg Does this image describe a place of hot spring? Please answer yes or no. Yes +Places365_val_00000179.jpg Does this image describe a place of mausoleum? Please answer yes or no. No +Places365_val_00000180.jpg Does this image describe a place of cottage? Please answer yes or no. Yes +Places365_val_00000180.jpg Does this image describe a place of parking garage indoor? Please answer yes or no. No +Places365_val_00000181.jpg Does this image describe a place of general store outdoor? Please answer yes or no. Yes +Places365_val_00000181.jpg Does this image describe a place of hotel room? Please answer yes or no. No +Places365_val_00000182.jpg Is this photo taken in a place of chalet? Please answer yes or no. Yes +Places365_val_00000182.jpg Is this photo taken in a place of dorm room? Please answer yes or no. No +Places365_val_00000183.jpg Is this photo taken in a place of flea market indoor? Please answer yes or no. Yes +Places365_val_00000183.jpg Is this photo taken in a place of tree house? Please answer yes or no. No +Places365_val_00000184.jpg Is this photo taken in a place of shower? Please answer yes or no. Yes +Places365_val_00000184.jpg Is this photo taken in a place of living room? Please answer yes or no. No +Places365_val_00000185.jpg Is this picture captured in a place of supermarket? Please answer yes or no. Yes +Places365_val_00000185.jpg Is this picture captured in a place of orchard? Please answer yes or no. No +Places365_val_00000186.jpg Is this picture captured in a place of bowling alley? Please answer yes or no. Yes +Places365_val_00000186.jpg Is this picture captured in a place of ice skating rink outdoor? Please answer yes or no. No +Places365_val_00000187.jpg Is this picture captured in a place of barn? Please answer yes or no. Yes +Places365_val_00000187.jpg Is this picture captured in a place of lobby? Please answer yes or no. No +Places365_val_00000188.jpg Is this photo taken in a place of japanese garden? Please answer yes or no. Yes +Places365_val_00000188.jpg Is this photo taken in a place of hangar indoor? Please answer yes or no. No +Places365_val_00000189.jpg Is this picture captured in a place of swimming hole? Please answer yes or no. Yes +Places365_val_00000189.jpg Is this picture captured in a place of bedroom? Please answer yes or no. No +Places365_val_00000190.jpg Is this picture captured in a place of pizzeria? Please answer yes or no. Yes +Places365_val_00000190.jpg Is this picture captured in a place of computer room? Please answer yes or no. No +Places365_val_00000191.jpg Does this image describe a place of volleyball court outdoor? Please answer yes or no. Yes +Places365_val_00000191.jpg Does this image describe a place of church indoor? Please answer yes or no. No +Places365_val_00000192.jpg Is this photo taken in a place of market indoor? Please answer yes or no. Yes +Places365_val_00000192.jpg Is this photo taken in a place of home office? Please answer yes or no. No +Places365_val_00000193.jpg Is this picture captured in a place of ice floe? Please answer yes or no. Yes +Places365_val_00000193.jpg Is this picture captured in a place of closet? Please answer yes or no. No +Places365_val_00000194.jpg Does this image describe a place of lake natural? Please answer yes or no. Yes +Places365_val_00000194.jpg Does this image describe a place of beer hall? Please answer yes or no. No +Places365_val_00000195.jpg Does this image describe a place of mountain path? Please answer yes or no. Yes +Places365_val_00000195.jpg Does this image describe a place of construction site? Please answer yes or no. No +Places365_val_00000196.jpg Is this photo taken in a place of orchestra pit? Please answer yes or no. Yes +Places365_val_00000196.jpg Is this photo taken in a place of burial chamber? Please answer yes or no. No +Places365_val_00000197.jpg Does this image describe a place of village? Please answer yes or no. Yes +Places365_val_00000197.jpg Does this image describe a place of underwater ocean deep? Please answer yes or no. No +Places365_val_00000198.jpg Does this image describe a place of waterfall? Please answer yes or no. Yes +Places365_val_00000198.jpg Does this image describe a place of booth indoor? Please answer yes or no. No +Places365_val_00000199.jpg Is this photo taken in a place of greenhouse indoor? Please answer yes or no. Yes +Places365_val_00000199.jpg Is this photo taken in a place of aqueduct? Please answer yes or no. No +Places365_val_00000200.jpg Is this photo taken in a place of television studio? Please answer yes or no. Yes +Places365_val_00000200.jpg Is this photo taken in a place of hunting lodge outdoor? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/Your_Results/text_translation.txt b/internvl_chat_audio/eval/mme/Your_Results/text_translation.txt new file mode 100644 index 00000000..8566fbaa --- /dev/null +++ b/internvl_chat_audio/eval/mme/Your_Results/text_translation.txt @@ -0,0 +1,40 @@ +0001.png Is it appropriate to translate the Chinese in the image into English 'classic taste' in the picture? Please answer yes or no. Yes +0001.png Is it appropriate to translate the Chinese in the image into English 'classic strawberry flavor' in the picture? Please answer yes or no. No +0002.png Is it appropriate to translate the Chinese in the image into English 'a delicious dinner' in the picture? Please answer yes or no. Yes +0002.png Is it appropriate to translate the Chinese in the image into English 'hamburger and chips' in the picture? Please answer yes or no. No +0003.png Is it appropriate to translate the Chinese in the image into English 'sunny weather' in the picture? Please answer yes or no. Yes +0003.png Is it appropriate to translate the Chinese in the image into English 'cold weather' in the picture? Please answer yes or no. No +0004.png Is it appropriate to translate the Chinese in the image into English 'run very fast' in the picture? Please answer yes or no. Yes +0004.png Is it appropriate to translate the Chinese in the image into English 'run very slow' in the picture? Please answer yes or no. No +0005.png Is it appropriate to translate the Chinese in the image into English 'feeling happy' in the picture? Please answer yes or no. Yes +0005.png Is it appropriate to translate the Chinese in the image into English 'feeling bored' in the picture? Please answer yes or no. No +0006.png Is it appropriate to translate the Chinese in the image into English 'work hard together' in the picture? Please answer yes or no. Yes +0006.png Is it appropriate to translate the Chinese in the image into English 'be filled with intrigue' in the picture? Please answer yes or no. No +0007.png Is it appropriate to translate the Chinese in the image into English 'walking very slowly' in the picture? Please answer yes or no. Yes +0007.png Is it appropriate to translate the Chinese in the image into English 'runing very slowly' in the picture? Please answer yes or no. No +0008.png Is it appropriate to translate the Chinese in the image into English 'very proud' in the picture? Please answer yes or no. Yes +0008.png Is it appropriate to translate the Chinese in the image into English 'very thankful' in the picture? Please answer yes or no. No +0009.png Is it appropriate to translate the Chinese in the image into English 'creative people' in the picture? Please answer yes or no. Yes +0009.png Is it appropriate to translate the Chinese in the image into English 'leading people' in the picture? Please answer yes or no. No +0010.png Is it appropriate to translate the Chinese in the image into English 'a beautiful garden' in the picture? Please answer yes or no. Yes +0010.png Is it appropriate to translate the Chinese in the image into English 'a beautiful campus' in the picture? Please answer yes or no. No +0011.png Is it appropriate to translate the Chinese in the image into English 'a difficult work' in the picture? Please answer yes or no. Yes +0011.png Is it appropriate to translate the Chinese in the image into English 'a easy work' in the picture? Please answer yes or no. No +0012.png Is it appropriate to translate the Chinese in the image into English 'a small amount' in the picture? Please answer yes or no. Yes +0012.png Is it appropriate to translate the Chinese in the image into English 'difficult and dangerous' in the picture? Please answer yes or no. No +0013.png Is it appropriate to translate the Chinese in the image into English 'feeling frustrated' in the picture? Please answer yes or no. Yes +0013.png Is it appropriate to translate the Chinese in the image into English 'feeling relaxed' in the picture? Please answer yes or no. No +0014.png Is it appropriate to translate the Chinese in the image into English 'waiting for a long time' in the picture? Please answer yes or no. Yes +0014.png Is it appropriate to translate the Chinese in the image into English 'sleeping for a long time' in the picture? Please answer yes or no. No +0015.png Is it appropriate to translate the Chinese in the image into English 'very powerful' in the picture? Please answer yes or no. Yes +0015.png Is it appropriate to translate the Chinese in the image into English 'to be fragile throughout the world' in the picture? Please answer yes or no. No +0016.png Is it appropriate to translate the Chinese in the image into English 'all talk and no action' in the picture? Please answer yes or no. Yes +0016.png Is it appropriate to translate the Chinese in the image into English 'hands-on practice' in the picture? Please answer yes or no. No +0017.png Is it appropriate to translate the Chinese in the image into English 'delicious fruit' in the picture? Please answer yes or no. Yes +0017.png Is it appropriate to translate the Chinese in the image into English 'banana' in the picture? Please answer yes or no. No +0018.png Is it appropriate to translate the Chinese in the image into English 'very unforgettable' in the picture? Please answer yes or no. Yes +0018.png Is it appropriate to translate the Chinese in the image into English 'very happy' in the picture? Please answer yes or no. No +0019.png Is it appropriate to translate the Chinese in the image into English 'get along well' in the picture? Please answer yes or no. Yes +0019.png Is it appropriate to translate the Chinese in the image into English 'for own self-interest' in the picture? Please answer yes or no. No +0020.png Is it appropriate to translate the Chinese in the image into English 'rank first' in the picture? Please answer yes or no. Yes +0020.png Is it appropriate to translate the Chinese in the image into English 'to add the finishing touches' in the picture? Please answer yes or no. No diff --git a/internvl_chat_audio/eval/mme/calculation.py b/internvl_chat_audio/eval/mme/calculation.py new file mode 100644 index 00000000..d10149ca --- /dev/null +++ b/internvl_chat_audio/eval/mme/calculation.py @@ -0,0 +1,162 @@ +import argparse +import os + +from sklearn.metrics import (accuracy_score, confusion_matrix, precision_score, + recall_score) + +parser = argparse.ArgumentParser() +parser.add_argument('--results_dir', default='./LaVIN', type=str) + +eval_type_dict = { + 'Perception': ['existence', 'count', 'position', 'color', 'posters', 'celebrity', 'scene', 'landmark', 'artwork', 'OCR'], + 'Cognition': ['commonsense_reasoning', 'numerical_calculation', 'text_translation', 'code_reasoning'] +} + + +class calculate_metrics: + def divide_chunks(self, l, n=2): + # looping till length l + for i in range(0, len(l), n): + yield l[i:i + n] + + return + + def parse_pred_ans(self, pred_ans): + pred_label = None + if pred_ans in ['yes', 'no']: + pred_label = pred_ans + else: + prefix_pred_ans = pred_ans[:4] + + if 'yes' in prefix_pred_ans: + pred_label = 'yes' + elif 'no' in prefix_pred_ans: + pred_label = 'no' + else: + pred_label = 'other' + + return pred_label + + def compute_metric(self, gts, preds): + assert len(gts) == len(preds) + + label_map = { + 'yes': 1, + 'no': 0, + 'other': -1, + } + + gts = [label_map[x] for x in gts] + preds = [label_map[x] for x in preds] + + acc = accuracy_score(gts, preds) + + clean_gts = [] + clean_preds = [] + other_num = 0 + for gt, pred in zip(gts, preds): + if pred == -1: + other_num += 1 + continue + clean_gts.append(gt) + clean_preds.append(pred) + + conf_mat = confusion_matrix(clean_gts, clean_preds, labels=[1,0]) + precision = precision_score(clean_gts, clean_preds, average='binary') + recall = recall_score(clean_gts, clean_preds, average='binary') + tp, fn = conf_mat[0] + fp, tn = conf_mat[1] + + metric_dict = dict() + metric_dict = { + 'TP': tp, + 'FN': fn, + 'TN': tn, + 'FP': fp, + 'precision': precision, + 'recall': recall, + 'other_num': other_num, + 'acc': acc, + } + + return metric_dict + + def process_result(self, results_dir): + + model_score_dict = dict() + for eval_type, task_name_list in eval_type_dict.items(): + print('===========', eval_type, '===========') + + scores = 0 + task_score_dict = dict() + + for task_name in task_name_list: + + task_txt = os.path.join(results_dir, task_name + '.txt') + lines = open(task_txt, 'r').readlines() + chunk_lines = list(self.divide_chunks(lines)) # one image corresponds to two questions + + img_num = len(chunk_lines) + task_other_ans_num = 0 + task_score = 0 + acc_plus_correct_num = 0 + gts = [] + preds = [] + + for img_items in chunk_lines: + assert len(img_items) == 2 + img_correct_num = 0 + + for img_item in img_items: + try: + img_name, question, gt_ans, pred_ans = img_item.split('\t') + except: + print(img_item) + continue + gt_ans = gt_ans.lower() + pred_ans = pred_ans.lower() + + assert gt_ans in ['yes', 'no'] # gt can only be yes or no. + + pred_ans = self.parse_pred_ans(pred_ans) + assert pred_ans in ['yes', 'no', 'other'] + + gts.append(gt_ans) + preds.append(pred_ans) + + if gt_ans == pred_ans: + img_correct_num += 1 + + if pred_ans not in ['yes', 'no']: + task_other_ans_num += 1 + + if img_correct_num == 2: + acc_plus_correct_num += 1 + + # cal TP precision acc, etc. + metric_dict = self.compute_metric(gts, preds) + acc_plus = acc_plus_correct_num / img_num + metric_dict['acc_plus'] = acc_plus + + for k, v in metric_dict.items(): + if k in ['acc', 'acc_plus']: + task_score += v*100 + + task_score_dict[task_name] = task_score + + scores += task_score + + print('total score:', scores, '\n') + for task_name, score in task_score_dict.items(): + print('\t', task_name, ' score:', score) + print('\n') + + return + + +if __name__ == '__main__': + cal = calculate_metrics() + + args = parser.parse_args() + results_dir = args.results_dir + cal.process_result(results_dir) diff --git a/internvl_chat_audio/eval/mme/eval.py b/internvl_chat_audio/eval/mme/eval.py new file mode 100644 index 00000000..de0d36bd --- /dev/null +++ b/internvl_chat_audio/eval/mme/eval.py @@ -0,0 +1,98 @@ +import argparse +import os +import re + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from tqdm import tqdm + + +def load_image(image_file, input_size=224): + image = Image.open(image_file).convert('RGB') + transform = build_transform(is_train=False, input_size=input_size) + if args.dynamic: + images = dynamic_preprocess(image, image_size=input_size, + use_thumbnail=use_thumbnail, + max_num=args.max_num) + else: + images = [image] + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + return pixel_values + + +def post_processing(response): + response = response.replace('\n', '').replace('不是', 'No').replace('是', 'Yes').replace('否', 'No') + response = response.lower().replace('true', 'yes').replace('false', 'no') + pattern = re.compile(r'[\u4e00-\u9fa5]') + response = re.sub(pattern, '', response) + return response + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--root', type=str, default='./Your_Results') + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--top-k', type=int, default=50) + parser.add_argument('--top-p', type=float, default=0.9) + parser.add_argument('--sample', type=bool, default=False) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + output = os.path.basename(args.checkpoint) + os.makedirs(output, exist_ok=True) + prompt = 'Answer the question using a single word or phrase.' + + for filename in os.listdir(args.root): + fin = open(os.path.join(args.root, filename), 'r', encoding='utf-8') + fout = open(os.path.join(output, filename), 'w', encoding='utf-8') + lines = fin.readlines() + filename = filename.replace('.txt', '') + for line in tqdm(lines): + img, question, gt = line.strip().split('\t') + question = question + ' ' + prompt + img_path = os.path.join('../../data/mme/MME_Benchmark_release_version', filename, img) + assert os.path.exists(img_path), img_path + pixel_values = load_image(img_path, image_size).cuda().to(torch.bfloat16) + generation_config = dict( + do_sample=args.sample, + top_k=args.top_k, + top_p=args.top_p, + num_beams=args.num_beams, + max_new_tokens=20, + eos_token_id=tokenizer.eos_token_id, + ) + response = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=question, + generation_config=generation_config, + verbose=True + ) + response = post_processing(response) + print(img, question, gt, response, sep='\t', file=fout) + fin.close() + fout.close() diff --git a/internvl_chat_audio/eval/mmmu/answer_dict_val.json b/internvl_chat_audio/eval/mmmu/answer_dict_val.json new file mode 100644 index 00000000..b1d0703e --- /dev/null +++ b/internvl_chat_audio/eval/mmmu/answer_dict_val.json @@ -0,0 +1,3611 @@ +{ + "validation_Accounting_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_2": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Accounting_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Accounting_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_7": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_9": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Accounting_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_11": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Accounting_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Accounting_17": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Accounting_18": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Accounting_19": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Accounting_20": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_23": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_26": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Accounting_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Accounting_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Accounting_30": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Agriculture_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Agriculture_2": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Agriculture_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Agriculture_4": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Agriculture_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Agriculture_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Agriculture_7": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Agriculture_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Agriculture_9": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Agriculture_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Agriculture_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Agriculture_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Agriculture_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Agriculture_14": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Agriculture_15": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Agriculture_16": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Agriculture_17": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Agriculture_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Agriculture_19": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Agriculture_20": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Agriculture_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Agriculture_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Agriculture_23": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Agriculture_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Agriculture_25": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Agriculture_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Agriculture_27": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Agriculture_28": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Agriculture_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Agriculture_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_2": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_4": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_8": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_10": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_12": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_14": { + "question_type": "short-answer", + "ground_truth": "1.06" + }, + "validation_Architecture_and_Engineering_15": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_16": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_17": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Architecture_and_Engineering_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_20": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_23": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Architecture_and_Engineering_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Architecture_and_Engineering_25": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Architecture_and_Engineering_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Architecture_and_Engineering_27": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Architecture_and_Engineering_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Architecture_and_Engineering_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_5": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_6": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_8": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_9": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_11": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_13": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_15": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_16": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_17": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_19": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_20": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_21": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_24": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_25": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_26": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_27": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_30": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_2": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_Theory_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_5": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_Theory_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_Theory_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_Theory_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_11": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_Theory_12": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_Theory_15": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_Theory_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_Theory_17": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_Theory_18": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_Theory_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_20": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_21": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_Theory_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_23": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_Theory_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Art_Theory_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Art_Theory_27": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Art_Theory_29": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Art_Theory_30": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Basic_Medical_Science_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Basic_Medical_Science_2": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Basic_Medical_Science_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Basic_Medical_Science_5": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_7": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_9": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Basic_Medical_Science_10": { + "question_type": "short-answer", + "ground_truth": "C" + }, + "validation_Basic_Medical_Science_11": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_12": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_14": { + "question_type": "short-answer", + "ground_truth": "Transformation" + }, + "validation_Basic_Medical_Science_15": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Basic_Medical_Science_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_20": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_23": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Basic_Medical_Science_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Basic_Medical_Science_26": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Basic_Medical_Science_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Basic_Medical_Science_29": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Basic_Medical_Science_30": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Biology_1": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Biology_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Biology_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Biology_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Biology_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Biology_7": { + "question_type": "multiple-choice", + "ground_truth": "I" + }, + "validation_Biology_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_9": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Biology_10": { + "question_type": "short-answer", + "ground_truth": "1/64" + }, + "validation_Biology_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Biology_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_14": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Biology_15": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Biology_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_19": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_20": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_21": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_23": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_25": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Biology_26": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Biology_27": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Biology_28": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_29": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Biology_30": { + "question_type": "multiple-choice", + "ground_truth": "F" + }, + "validation_Chemistry_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Chemistry_4": { + "question_type": "short-answer", + "ground_truth": "trans-1-Chloro-4-methylcyclohexane" + }, + "validation_Chemistry_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Chemistry_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_7": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Chemistry_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Chemistry_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Chemistry_10": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Chemistry_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Chemistry_13": { + "question_type": "short-answer", + "ground_truth": "12.97" + }, + "validation_Chemistry_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_15": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_16": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Chemistry_17": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_18": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Chemistry_19": { + "question_type": "short-answer", + "ground_truth": "4" + }, + "validation_Chemistry_20": { + "question_type": "short-answer", + "ground_truth": "5" + }, + "validation_Chemistry_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Chemistry_23": { + "question_type": "short-answer", + "ground_truth": "6.5" + }, + "validation_Chemistry_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Chemistry_25": { + "question_type": "short-answer", + "ground_truth": "24.32" + }, + "validation_Chemistry_26": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Chemistry_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Chemistry_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Chemistry_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Chemistry_30": { + "question_type": "short-answer", + "ground_truth": [ + "$MgS$", + "MgS" + ] + }, + "validation_Clinical_Medicine_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Clinical_Medicine_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Clinical_Medicine_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_4": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_5": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Clinical_Medicine_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Clinical_Medicine_7": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Clinical_Medicine_8": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_9": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_10": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Clinical_Medicine_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Clinical_Medicine_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Clinical_Medicine_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Clinical_Medicine_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_17": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Clinical_Medicine_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Clinical_Medicine_19": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Clinical_Medicine_20": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Clinical_Medicine_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Clinical_Medicine_22": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Clinical_Medicine_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Clinical_Medicine_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Clinical_Medicine_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Clinical_Medicine_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Clinical_Medicine_27": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Clinical_Medicine_28": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Clinical_Medicine_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Clinical_Medicine_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_2": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_4": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_5": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_7": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_9": { + "question_type": "short-answer", + "ground_truth": "60" + }, + "validation_Computer_Science_10": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_11": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_12": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_13": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_14": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_15": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_16": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_17": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_19": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_20": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_21": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Computer_Science_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_23": { + "question_type": "short-answer", + "ground_truth": "253.75" + }, + "validation_Computer_Science_24": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Computer_Science_25": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Computer_Science_26": { + "question_type": "short-answer", + "ground_truth": "251" + }, + "validation_Computer_Science_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Computer_Science_29": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Computer_Science_30": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Design_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Design_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Design_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_4": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Design_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_8": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Design_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_10": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Design_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Design_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Design_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Design_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_17": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Design_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Design_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_20": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Design_21": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Design_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Design_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Design_25": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Design_26": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Design_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Design_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Design_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Design_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Diagnostics_and_Laboratory_Medicine_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Diagnostics_and_Laboratory_Medicine_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Diagnostics_and_Laboratory_Medicine_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Diagnostics_and_Laboratory_Medicine_8": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Diagnostics_and_Laboratory_Medicine_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Diagnostics_and_Laboratory_Medicine_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Diagnostics_and_Laboratory_Medicine_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_12": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Diagnostics_and_Laboratory_Medicine_13": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Diagnostics_and_Laboratory_Medicine_14": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Diagnostics_and_Laboratory_Medicine_15": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Diagnostics_and_Laboratory_Medicine_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Diagnostics_and_Laboratory_Medicine_17": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Diagnostics_and_Laboratory_Medicine_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_19": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Diagnostics_and_Laboratory_Medicine_20": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Diagnostics_and_Laboratory_Medicine_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Diagnostics_and_Laboratory_Medicine_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Diagnostics_and_Laboratory_Medicine_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Diagnostics_and_Laboratory_Medicine_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_26": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_27": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Diagnostics_and_Laboratory_Medicine_28": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Diagnostics_and_Laboratory_Medicine_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Diagnostics_and_Laboratory_Medicine_30": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Economics_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_6": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Economics_7": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Economics_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_13": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Economics_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Economics_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_20": { + "question_type": "short-answer", + "ground_truth": "8" + }, + "validation_Economics_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_23": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Economics_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Economics_25": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Economics_26": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_27": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_29": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Economics_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Electronics_2": { + "question_type": "short-answer", + "ground_truth": "2.83" + }, + "validation_Electronics_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_4": { + "question_type": "short-answer", + "ground_truth": "8.4" + }, + "validation_Electronics_5": { + "question_type": "short-answer", + "ground_truth": "62.6" + }, + "validation_Electronics_6": { + "question_type": "short-answer", + "ground_truth": "71.6" + }, + "validation_Electronics_7": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Electronics_9": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Electronics_11": { + "question_type": "short-answer", + "ground_truth": "0.3" + }, + "validation_Electronics_12": { + "question_type": "short-answer", + "ground_truth": "10" + }, + "validation_Electronics_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Electronics_16": { + "question_type": "short-answer", + "ground_truth": "20" + }, + "validation_Electronics_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Electronics_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Electronics_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Electronics_20": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_21": { + "question_type": "short-answer", + "ground_truth": "0.9965" + }, + "validation_Electronics_22": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Electronics_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Electronics_24": { + "question_type": "short-answer", + "ground_truth": "551" + }, + "validation_Electronics_25": { + "question_type": "short-answer", + "ground_truth": "50" + }, + "validation_Electronics_26": { + "question_type": "short-answer", + "ground_truth": "6.333" + }, + "validation_Electronics_27": { + "question_type": "short-answer", + "ground_truth": "-120" + }, + "validation_Electronics_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Electronics_29": { + "question_type": "short-answer", + "ground_truth": "30" + }, + "validation_Electronics_30": { + "question_type": "short-answer", + "ground_truth": "-141" + }, + "validation_Energy_and_Power_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_2": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_7": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_8": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_9": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_12": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_20": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Energy_and_Power_25": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_26": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Energy_and_Power_29": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Energy_and_Power_30": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Finance_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Finance_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Finance_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Finance_4": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Finance_5": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Finance_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Finance_7": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Finance_8": { + "question_type": "short-answer", + "ground_truth": "360" + }, + "validation_Finance_9": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Finance_10": { + "question_type": "short-answer", + "ground_truth": "2000000" + }, + "validation_Finance_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Finance_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Finance_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Finance_14": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Finance_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Finance_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Finance_17": { + "question_type": "short-answer", + "ground_truth": "1000" + }, + "validation_Finance_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Finance_19": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Finance_20": { + "question_type": "short-answer", + "ground_truth": "7243000" + }, + "validation_Finance_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Finance_22": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Finance_23": { + "question_type": "short-answer", + "ground_truth": "527.89" + }, + "validation_Finance_24": { + "question_type": "short-answer", + "ground_truth": "30.0" + }, + "validation_Finance_25": { + "question_type": "short-answer", + "ground_truth": "0.286" + }, + "validation_Finance_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Finance_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Finance_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Finance_29": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Finance_30": { + "question_type": "short-answer", + "ground_truth": "1249" + }, + "validation_Geography_1": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Geography_4": { + "question_type": "short-answer", + "ground_truth": [ + "Tampa", + "Florida" + ] + }, + "validation_Geography_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Geography_6": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Geography_7": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Geography_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Geography_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Geography_11": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Geography_13": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_14": { + "question_type": "short-answer", + "ground_truth": "8200" + }, + "validation_Geography_15": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Geography_17": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Geography_19": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_20": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Geography_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Geography_22": { + "question_type": "short-answer", + "ground_truth": "3" + }, + "validation_Geography_23": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Geography_25": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Geography_26": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Geography_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Geography_28": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Geography_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Geography_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_History_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_History_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_History_5": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_History_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_8": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_History_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_History_11": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_History_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_13": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_15": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_History_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_17": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_History_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_19": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_History_20": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_22": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_23": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_History_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_History_25": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_26": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_History_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_29": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_History_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Literature_3": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Literature_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_10": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_13": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Literature_14": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_17": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_20": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_23": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Literature_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Literature_26": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Literature_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_28": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Literature_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Literature_30": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Manage_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Manage_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_3": { + "question_type": "short-answer", + "ground_truth": "242110.62" + }, + "validation_Manage_4": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Manage_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_6": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Manage_7": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_8": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Manage_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Manage_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Manage_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Manage_14": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Manage_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Manage_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Manage_17": { + "question_type": "short-answer", + "ground_truth": "65" + }, + "validation_Manage_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_19": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Manage_20": { + "question_type": "short-answer", + "ground_truth": "2960" + }, + "validation_Manage_21": { + "question_type": "short-answer", + "ground_truth": "9.20" + }, + "validation_Manage_22": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Manage_23": { + "question_type": "short-answer", + "ground_truth": "1464" + }, + "validation_Manage_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Manage_25": { + "question_type": "short-answer", + "ground_truth": "0.963" + }, + "validation_Manage_26": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Manage_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Manage_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Manage_30": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Marketing_1": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Marketing_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_3": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Marketing_4": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Marketing_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Marketing_7": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Marketing_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Marketing_9": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Marketing_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_11": { + "question_type": "short-answer", + "ground_truth": "0.10" + }, + "validation_Marketing_12": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Marketing_13": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Marketing_14": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Marketing_15": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Marketing_17": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Marketing_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Marketing_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_20": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Marketing_21": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_22": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Marketing_23": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Marketing_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Marketing_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_26": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Marketing_27": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Marketing_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Materials_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_3": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Materials_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_6": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Materials_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Materials_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Materials_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Materials_11": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Materials_12": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_13": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Materials_14": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Materials_15": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Materials_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Materials_17": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Materials_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Materials_20": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Materials_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Materials_22": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Materials_23": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Materials_24": { + "question_type": "multiple-choice", + "ground_truth": "F" + }, + "validation_Materials_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Materials_26": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Materials_27": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Materials_28": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Materials_29": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Materials_30": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Math_2": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Math_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Math_4": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_5": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Math_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Math_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Math_10": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Math_11": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Math_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Math_15": { + "question_type": "short-answer", + "ground_truth": [ + "24/7", + "3.429" + ] + }, + "validation_Math_16": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Math_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_20": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Math_21": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Math_22": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Math_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_25": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Math_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Math_27": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_29": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Math_30": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Mechanical_Engineering_1": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Mechanical_Engineering_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Mechanical_Engineering_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_5": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Mechanical_Engineering_6": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Mechanical_Engineering_7": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_9": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Mechanical_Engineering_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_11": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Mechanical_Engineering_12": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Mechanical_Engineering_13": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Mechanical_Engineering_14": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Mechanical_Engineering_15": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Mechanical_Engineering_16": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Mechanical_Engineering_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_20": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Mechanical_Engineering_21": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Mechanical_Engineering_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_23": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_24": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_26": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Mechanical_Engineering_28": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Mechanical_Engineering_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Mechanical_Engineering_30": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Music_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Music_2": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Music_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_7": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_9": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Music_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_11": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Music_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Music_13": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Music_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Music_16": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_17": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_20": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Music_21": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Music_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Music_23": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Music_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Music_25": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_26": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_27": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Music_28": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Music_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Music_30": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Pharmacy_2": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Pharmacy_3": { + "question_type": "short-answer", + "ground_truth": "2" + }, + "validation_Pharmacy_4": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_5": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Pharmacy_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Pharmacy_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_10": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Pharmacy_11": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Pharmacy_14": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Pharmacy_15": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Pharmacy_17": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Pharmacy_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_19": { + "question_type": "short-answer", + "ground_truth": "B" + }, + "validation_Pharmacy_20": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Pharmacy_21": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Pharmacy_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_23": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Pharmacy_24": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Pharmacy_25": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Pharmacy_26": { + "question_type": "short-answer", + "ground_truth": "A" + }, + "validation_Pharmacy_27": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Pharmacy_28": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Pharmacy_29": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Pharmacy_30": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_2": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Physics_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_5": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Physics_6": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Physics_7": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Physics_8": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_9": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Physics_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Physics_12": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Physics_14": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_15": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_16": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Physics_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_18": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_20": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_21": { + "question_type": "short-answer", + "ground_truth": "A" + }, + "validation_Physics_22": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_23": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Physics_24": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Physics_25": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_27": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Physics_28": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_29": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Physics_30": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Psychology_1": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_2": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_3": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Psychology_4": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Psychology_5": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Psychology_6": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Psychology_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_10": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Psychology_11": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Psychology_12": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Psychology_13": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Psychology_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Psychology_15": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Psychology_16": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Psychology_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_18": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Psychology_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Psychology_20": { + "question_type": "short-answer", + "ground_truth": "embryonic" + }, + "validation_Psychology_21": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Psychology_22": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Psychology_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Psychology_25": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Psychology_26": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Psychology_27": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Psychology_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_29": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Psychology_30": { + "question_type": "multiple-choice", + "ground_truth": "E" + }, + "validation_Public_Health_1": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Public_Health_2": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_3": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_4": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Public_Health_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Public_Health_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Public_Health_7": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_8": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Public_Health_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_10": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_11": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Public_Health_12": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Public_Health_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Public_Health_15": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Public_Health_16": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_17": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Public_Health_18": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Public_Health_19": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Public_Health_20": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Public_Health_21": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Public_Health_22": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_23": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Public_Health_25": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Public_Health_26": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Public_Health_27": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Public_Health_28": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Public_Health_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Public_Health_30": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Sociology_1": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_2": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_3": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_4": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Sociology_5": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Sociology_6": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_7": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_8": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_9": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Sociology_10": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Sociology_11": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_12": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Sociology_13": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_14": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_15": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Sociology_16": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_17": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Sociology_18": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_19": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Sociology_20": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_21": { + "question_type": "multiple-choice", + "ground_truth": "C" + }, + "validation_Sociology_22": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_23": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_24": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_25": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_26": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_27": { + "question_type": "multiple-choice", + "ground_truth": "D" + }, + "validation_Sociology_28": { + "question_type": "multiple-choice", + "ground_truth": "A" + }, + "validation_Sociology_29": { + "question_type": "multiple-choice", + "ground_truth": "B" + }, + "validation_Sociology_30": { + "question_type": "multiple-choice", + "ground_truth": "E" + } +} diff --git a/internvl_chat_audio/eval/mmmu/data_utils.py b/internvl_chat_audio/eval/mmmu/data_utils.py new file mode 100644 index 00000000..0c9823d8 --- /dev/null +++ b/internvl_chat_audio/eval/mmmu/data_utils.py @@ -0,0 +1,176 @@ +"""Utils for data load, save, and process (e.g., prompt construction)""" + +import json +import os +import re + +import yaml + +DOMAIN_CAT2SUB_CAT = { + 'Art and Design': ['Art', 'Art_Theory', 'Design', 'Music'], + 'Business': ['Accounting', 'Economics', 'Finance', 'Manage', 'Marketing'], + 'Science': ['Biology', 'Chemistry', 'Geography', 'Math', 'Physics', ], + 'Health and Medicine': ['Basic_Medical_Science', 'Clinical_Medicine', 'Diagnostics_and_Laboratory_Medicine', + 'Pharmacy', 'Public_Health'], + 'Humanities and Social Science': ['History', 'Literature', 'Sociology', 'Psychology'], + 'Tech and Engineering': ['Agriculture', 'Architecture_and_Engineering', 'Computer_Science', 'Electronics', + 'Energy_and_Power', 'Materials', 'Mechanical_Engineering'], +} + +CAT_SHORT2LONG = { + 'acc': 'Accounting', + 'agri': 'Agriculture', + 'arch': 'Architecture_and_Engineering', + 'art': 'Art', + 'art_theory': 'Art_Theory', + 'bas_med': 'Basic_Medical_Science', + 'bio': 'Biology', + 'chem': 'Chemistry', + 'cli_med': 'Clinical_Medicine', + 'cs': 'Computer_Science', + 'design': 'Design', + 'diag_med': 'Diagnostics_and_Laboratory_Medicine', + 'econ': 'Economics', + 'elec': 'Electronics', + 'ep': 'Energy_and_Power', + 'fin': 'Finance', + 'geo': 'Geography', + 'his': 'History', + 'liter': 'Literature', + 'manage': 'Manage', + 'mark': 'Marketing', + 'mate': 'Materials', + 'math': 'Math', + 'mech': 'Mechanical_Engineering', + 'music': 'Music', + 'phar': 'Pharmacy', + 'phys': 'Physics', + 'psy': 'Psychology', + 'pub_health': 'Public_Health', + 'socio': 'Sociology' +} + + +# DATA SAVING +def save_json(filename, ds): + with open(filename, 'w') as f: + json.dump(ds, f, indent=4) + + +def get_multi_choice_info(options): + """ + Given the list of options for multiple choice question + Return the index2ans and all_choices + """ + + start_chr = 'A' + all_choices = [] + index2ans = {} + for i, option in enumerate(options): + index2ans[chr(ord(start_chr) + i)] = option + all_choices.append(chr(ord(start_chr) + i)) + + return index2ans, all_choices + + +def load_yaml(file_path): + with open(file_path, 'r') as stream: + try: + yaml_dict = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + return yaml_dict + + +def parse_img_path(text): + matches = re.findall("", text) + return matches + + +def process_single_sample(data): + question = data['question'] + o_imgs_paths = [] + for option in data['options']: + current_o_imgs_paths = parse_img_path(option) + for img_path in current_o_imgs_paths: + o_imgs_paths.append(img_path) + images = [data['image_1'], data['image_2'], data['image_3'], data['image_4'], + data['image_5'], data['image_6'], data['image_7']] + return {'id': data['id'], 'question': question, 'options': data['options'], 'answer': data['answer'], + 'image': images, 'question_type': data['question_type']} + + +# DATA SAVING +def save_json(filename, ds): + with open(filename, 'w') as f: + json.dump(ds, f, indent=4) + + +def save_jsonl(filename, data): + """ + Save a dictionary of data to a JSON Lines file with the filename as key and caption as value. + + Args: + filename (str): The path to the file where the data should be saved. + data (dict): The dictionary containing the data to save where key is the image path and value is the caption. + """ + with open(filename, 'w', encoding='utf-8') as f: + for img_path, caption in data.items(): + # Extract the base filename without the extension + base_filename = os.path.basename(img_path) + # Create a JSON object with the filename as the key and caption as the value + json_record = json.dumps({base_filename: caption}, ensure_ascii=False) + # Write the JSON object to the file, one per line + f.write(json_record + '\n') + + +def save_args(args, path_dir): + argsDict = args.__dict__ + with open(path_dir + 'setting.txt', 'w') as f: + f.writelines('------------------ start ------------------' + '\n') + for eachArg, value in argsDict.items(): + f.writelines(eachArg + ' : ' + str(value) + '\n') + f.writelines('------------------- end -------------------') + + +# DATA PROCESSING +def construct_prompt(sample, config): + question = sample['question'] + options = eval(sample['options']) + example = '' + if sample['question_type'] == 'multiple-choice': + start_chr = 'A' + prediction_range = [] + index2ans = {} + for option in options: + prediction_range.append(start_chr) + example += f'({start_chr}) {option}\n' + index2ans[start_chr] = option + start_chr = chr(ord(start_chr) + 1) + empty_prompt_sample_structure = config['multi_choice_example_format'] + empty_prompt = empty_prompt_sample_structure.format(question, example) + res_dict = {} + res_dict['index2ans'] = index2ans + res_dict['correct_choice'] = sample['answer'] + res_dict['all_choices'] = prediction_range + res_dict['empty_prompt'] = empty_prompt + if config['task_instructions']: + res_dict['final_input_prompt'] = config['task_instructions'].strip() + '\n\n' + empty_prompt + else: + res_dict['final_input_prompt'] = empty_prompt + + res_dict['gt_content'] = options[ord(sample['answer'].upper()) - ord('A')] + else: + empty_prompt_sample_structure = config['short_ans_example_format'] + empty_prompt = empty_prompt_sample_structure.format(question) + res_dict = {} + res_dict['empty_prompt'] = empty_prompt + if config['task_instructions']: + res_dict['final_input_prompt'] = config['task_instructions'].strip() + '\n\n' + empty_prompt + else: + res_dict['final_input_prompt'] = empty_prompt + res_dict['gt_content'] = sample['answer'] + + res_dict.update(sample) + return res_dict diff --git a/internvl_chat_audio/eval/mmmu/eval_utils.py b/internvl_chat_audio/eval/mmmu/eval_utils.py new file mode 100644 index 00000000..754c080c --- /dev/null +++ b/internvl_chat_audio/eval/mmmu/eval_utils.py @@ -0,0 +1,264 @@ +"""Response Parsing and Evaluation for various models""" +import random +import re +from typing import Dict + +random.seed(42) +import numpy as np + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [',', '.', '!', '?', ';', ':', "'"]: + response = response.strip(char) + response = ' ' + response + ' ' # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f'({choice})' in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f' {choice} ' in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f'({can})') + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f' {can} ') + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(',', '')) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(',', '') + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [' ' + string, string + ' '] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r'-?\b\d{1,3}(?:,\d{3})+\b' + # Pattern for scientific notation + pattern_scientific = r'-?\d+(?:\.\d+)?[eE][+-]?\d+' + # Pattern for simple numbers without commas + pattern_simple = r'-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])' + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip('.').lower() + sub_responses = re.split(r'\.\s(?=[A-Z])|\n', response) + indicators_of_keys = ['could be ', 'so ', 'is ', + 'thus ', 'therefore ', 'final ', 'answer ', 'result '] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(['=']) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [':', ',', '.', '!', '?', ';', ':', "'"]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct + + +# ----------- Batch Evaluation ------------- +def evaluate(samples): + """ + Batch evaluation for multiple choice and open questions. + """ + pred_correct = 0 + judge_dict = dict() + for sample in samples: + gold_i = sample['answer'] + pred_i = sample['parsed_pred'] + if sample['question_type'] == 'multiple-choice': + correct = eval_multi_choice(gold_i, pred_i) + else: # open question + correct = eval_open(gold_i, pred_i) + + if correct: + judge_dict[sample['id']] = 'Correct' + pred_correct += 1 + else: + judge_dict[sample['id']] = 'Wrong' + + if len(samples) == 0: + return {'acc': 0} + return judge_dict, {'acc': pred_correct / len(samples)} + + +# ----------- Calculate Accuracy ------------- +def calculate_ins_level_acc(results: Dict): + """Calculate the instruction level accuracy for given Subject results""" + acc = 0 + ins_num = 0 + for cat_results in results.values(): + acc += cat_results['acc'] * cat_results['num_example'] + ins_num += cat_results['num_example'] + if ins_num == 0: + return 0 + return acc / ins_num diff --git a/internvl_chat_audio/eval/mmmu/evaluate_mmmu.py b/internvl_chat_audio/eval/mmmu/evaluate_mmmu.py new file mode 100644 index 00000000..57bd6ca0 --- /dev/null +++ b/internvl_chat_audio/eval/mmmu/evaluate_mmmu.py @@ -0,0 +1,304 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from data_utils import CAT_SHORT2LONG, process_single_sample +from datasets import concatenate_datasets, load_dataset +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from torch.utils.data import Dataset +from tqdm import tqdm + +ds_collections = { + 'MMMU_validation': { + 'root': 'MMMU/MMMU', + 'max_new_tokens': 10, + 'min_new_tokens': 1, + 'split': 'validation' + }, + 'MMMU_test': { + 'root': 'MMMU/MMMU', + 'max_new_tokens': 10, + 'min_new_tokens': 1, + 'split': 'test' + }, + 'MMMU_dev': { + 'root': 'MMMU/MMMU', + 'max_new_tokens': 10, + 'min_new_tokens': 1, + 'split': 'dev' + }, +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + answers = [_['answer'] for _ in batches] + data_ids = [_['data_id'] for _ in batches] + options = [_['option'] for _ in batches] + return pixel_values, questions, answers, data_ids, options + + +class MMMUDataset(torch.utils.data.Dataset): + + def __init__(self, root, split, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + # run for each subject + sub_dataset_list = [] + for subject in tqdm(CAT_SHORT2LONG.values()): + sub_dataset = load_dataset(root, subject, split=split, cache_dir=os.path.join(os.getcwd(), 'data/MMMU/')) + sub_dataset_list.append(sub_dataset) + + # merge all dataset + self.data = concatenate_datasets(sub_dataset_list) + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + + data = process_single_sample(self.data[idx]) + data_id = data['id'] + question = data['question'].strip() + pil_images = data['image'] + question_type = data['question_type'] + + choices = eval(data['options']) + answer = data['answer'] if 'answer' in data else None + + choice_list = [] + options = {} + multiple_choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M'] + for i, c in enumerate(choices): + choice_list.append('{}. {}'.format(multiple_choices[i], c.strip())) + options[multiple_choices[i]] = c.strip() + choice_txt = '\n'.join(choice_list) + if self.dynamic_image_size: + images = [] + for idx, pil_image in enumerate(pil_images): + if pil_image is not None: + if idx == 0: + pil_image = pil_image.resize((pil_image.width * 2, pil_image.height * 2), Image.BILINEAR) + pil_image = dynamic_preprocess(pil_image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, max_num=self.max_num) + else: + pil_image = dynamic_preprocess(pil_image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, max_num=1) + images += pil_image + else: + images = [pil_images[0]] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + if len(choice_txt) > 0: + question += '\n' + choice_txt + question += '\n' + self.prompt[question_type] + question = question.strip() + + return { + 'question': question, + 'pixel_values': pixel_values, + 'answer': answer, + 'option': options, + 'data_id': data_id + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def post_process(pred, option): + pred = pred.strip() + option_candidate = list(option.keys()) + if len(pred) == 1: + return pred + elif len(pred) != 1 and pred[0] in option_candidate: + return pred[0] + elif len(pred) != 1 and pred[0] not in option_candidate: + for k, v in option.items(): + if v in pred: + return k + + return pred + + +def evaluate_chat_model(): + prompt = { + 'multiple-choice': "Answer with the option's letter from the given choices directly.", + 'open': 'Answer the question using a single word or phrase.' + } + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = MMMUDataset( + root=ds_collections[ds_name]['root'], + split=ds_collections[ds_name]['split'], + prompt=prompt, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, answers, data_ids, options) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + if len(options[0]) == 0: + preds = [pred] + else: + preds = [post_process(pred, options[0])] + + for question, pred, answer, data_id in zip(questions, preds, answers, data_ids): + outputs.append({ + 'question': question, + 'answer': pred, + 'gt_answers': answer, + 'data_id': data_id + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + output_path = os.path.join(args.out_dir, results_file) + outputs = {} + for item in merged_outputs: + outputs[item['data_id']] = item['answer'] + with open(output_path, 'w') as f: + json.dump(outputs, f, indent=4) + print('Results saved to {}'.format(output_path)) + if ds_collections[ds_name]['split'] == 'validation': + print('Evaluating ...') + cmd = f'python eval/mmmu/main_eval_only.py ' \ + f'--output_path {output_path} ' \ + f'--answer_path eval/mmmu/answer_dict_val.json' + print(cmd) + os.system(cmd) + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.jsonl' + output_path = os.path.join(args.out_dir, results_file) + writer = open(output_path, 'w') + for item in merged_outputs: + writer.write(json.dumps(item) + '\n') + writer.close() + print('Results saved to {}'.format(output_path)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='MMMU_dev') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/mmmu/main_eval_only.py b/internvl_chat_audio/eval/mmmu/main_eval_only.py new file mode 100644 index 00000000..a1af8d29 --- /dev/null +++ b/internvl_chat_audio/eval/mmmu/main_eval_only.py @@ -0,0 +1,95 @@ +"""Parse and Evalate""" +import json +from argparse import ArgumentParser + +from data_utils import CAT_SHORT2LONG, DOMAIN_CAT2SUB_CAT, save_json +from eval_utils import (calculate_ins_level_acc, evaluate, + parse_multi_choice_response, parse_open_response) + +if __name__ == '__main__': + + parser = ArgumentParser() + parser.add_argument('--output_path', type=str, default='./example_outputs/qwen_vl/total_val_output.json', + help='The path to model output file.') + parser.add_argument('--answer_path', type=str, default='./answer_dict_val.json', help='Answer file path.') + args = parser.parse_args() + + output_dict = json.load(open(args.output_path)) + answer_dict = json.load(open(args.answer_path)) + + # group by category + output_dict_w_cat = {} + for data_id, parsed_pred in output_dict.items(): + category = '_'.join(data_id.split('_')[1:-1]) + if category not in output_dict_w_cat: + output_dict_w_cat.update({category: {}}) + output_dict_w_cat[category].update({data_id: parsed_pred}) + + # group by category + answer_dict_w_cat = {} + for data_id, parsed_pred in answer_dict.items(): + category = '_'.join(data_id.split('_')[1:-1]) + if category not in answer_dict_w_cat: + answer_dict_w_cat.update({category: {}}) + answer_dict_w_cat[category].update({data_id: parsed_pred}) + + evaluation_result = {} + + for category in CAT_SHORT2LONG.values(): + print('Evaluating: {}'.format(category)) + # get cat_outputs and cat_answers + try: + cat_outputs = output_dict_w_cat[category] + cat_answers = answer_dict_w_cat[category] + except KeyError: + print('Skipping {} for not found'.format(category)) + continue + + exampels_to_eval = [] + for data_id, parsed_pred in cat_outputs.items(): + question_type = cat_answers[data_id]['question_type'] + if question_type != 'multiple-choice': + parsed_pred = parse_open_response(parsed_pred) # mainly for type consistency (make it number, etc.) + else: + parsed_pred = parsed_pred + + exampels_to_eval.append({ + 'id': data_id, + 'question_type': question_type, + 'answer': cat_answers[data_id]['ground_truth'], + 'parsed_pred': parsed_pred + }) + + judge_dict, metric_dict = evaluate(exampels_to_eval) + metric_dict.update({'num_example': len(exampels_to_eval)}) + + evaluation_result[category] = metric_dict + + printable_results = {} + # pdb.set_trace() + # add domain Subject + for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): + in_domain_cat_results = {} + for cat_name in in_domain_cats: # use the order in DOMAIN_CAT2SUB_CAT + if cat_name in evaluation_result.keys(): + in_domain_cat_results[cat_name] = evaluation_result[cat_name] + else: + pass + in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) + in_domain_data_num = sum([cat_results['num_example'] for cat_results in in_domain_cat_results.values()]) + printable_results['Overall-' + domain] = {'num': int(in_domain_data_num), + 'acc': round(in_domain_ins_acc, 3) + } + # add sub category + for cat_name, cat_results in in_domain_cat_results.items(): + printable_results[cat_name] = {'num': int(cat_results['num_example']), + 'acc': round(cat_results['acc'], 3) + } + + # table.append(["-----------------------------", "-----", "----"]) + all_ins_acc = calculate_ins_level_acc(evaluation_result) + printable_results['Overall'] = { + 'num': sum([cat_results['num_example'] for cat_results in evaluation_result.values()]), + 'acc': round(all_ins_acc, 3)} + + print(printable_results) diff --git a/internvl_chat_audio/eval/mmvet/evaluate_mmvet.py b/internvl_chat_audio/eval/mmvet/evaluate_mmvet.py new file mode 100644 index 00000000..e9001f32 --- /dev/null +++ b/internvl_chat_audio/eval/mmvet/evaluate_mmvet.py @@ -0,0 +1,150 @@ +import argparse +import json +import os +import random +import time + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from tqdm import tqdm + +ds_collections = { + 'mmvet': { + 'root': 'data/mm-vet/images', + 'question': 'data/mm-vet/llava-mm-vet.jsonl', + 'metric': None, + 'max_new_tokens': 1000, + 'min_new_tokens': 1, + } +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + question_ids = [_['question_id'] for _ in batches] + annotations = [_['annotation'] for _ in batches] + + return pixel_values, questions, question_ids, annotations + + +class VQADataset(torch.utils.data.Dataset): + + def __init__(self, root, data, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + self.root = root + self.data = open(data).readlines() + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + data = json.loads(self.data[idx].strip()) + image, question, question_id, annotation = data['image'], data[ + 'text'], data['question_id'], data.get('answer', None) + + image = os.path.join(self.root, image) + image = Image.open(image).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + question = question + ' ' + self.prompt + return question_id, question, pixel_values, annotation + + +def evaluate_chat_model(): + random.seed(args.seed) + prompt = '' + + for ds_name in args.datasets: + dataset = VQADataset( + root=ds_collections[ds_name]['root'], + data=ds_collections[ds_name]['question'], + prompt=prompt, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + + outputs = {} + for _, (question_id, question, pixel_values, annotations) in tqdm(enumerate(dataset)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=question, + generation_config=generation_config, + verbose=True + ) + + outputs[f'v1_{question_id}'] = pred + + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + results_file = os.path.join(args.out_dir, results_file) + json.dump(outputs, open(results_file, 'w')) + print('Results saved to {}'.format(results_file)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='pope') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/mmvp/evaluate_mmvp.py b/internvl_chat_audio/eval/mmvp/evaluate_mmvp.py new file mode 100644 index 00000000..0314fa98 --- /dev/null +++ b/internvl_chat_audio/eval/mmvp/evaluate_mmvp.py @@ -0,0 +1,286 @@ +import argparse +import csv +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from torch.utils.data import Dataset +from tqdm import tqdm + +ds_collections = { + 'MMVP': { + 'root': 'data/MMVP', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + }, +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + answers = [_['answer'] for _ in batches] + data_ids = [_['data_id'] for _ in batches] + options = [_['option'] for _ in batches] + return pixel_values, questions, answers, data_ids, options + + +class MMVPDataset(torch.utils.data.Dataset): + + def __init__(self, root, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + # run for each subject + meta_path = os.path.join(root, 'Questions.csv') + with open(meta_path, 'r') as file: + csv_reader = csv.DictReader(file) + data = [row for row in csv_reader] + + self.data = data + self.root = root + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + data = self.data[idx] + # {'lndex': '1', 'Question': "Are the butterfly's wings closer to being open or closed?", + # 'Options': '(a) Open (b) Closed', 'Correct Answer': '(a)'} + data_id = data['lndex'] + question = data['Question'] + # question = '\n' + question + image = os.path.join(self.root + '/MMVP Images', data_id + '.jpg') + image = Image.open(image).convert('RGB') + + options = data['Options'].split('(b)') + options[0] = options[0].replace('(a)', '').strip() + options[1] = options[1].replace('(b)', '').strip() + + answer = data['Correct Answer'] if 'Correct Answer' in data else None + answer = answer.replace('(a)', 'A').replace('(b)', 'B').replace('(c)', 'C').replace('(d)', 'D') + choice_list = [] + new_options = {} + multiple_choices = ['A', 'B', 'C', 'D'] + for i, c in enumerate(options): + choice_list.append('{}. {}'.format(multiple_choices[i], c.strip())) + new_options[multiple_choices[i]] = c.strip() + choice_txt = '\n'.join(choice_list) + + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + if len(choice_txt) > 0: + question += '\n' + choice_txt + question += '\n' + self.prompt + print(question, answer) + return { + 'question': question, + 'pixel_values': pixel_values, + 'answer': answer, + 'option': new_options, + 'data_id': data_id + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def post_process(pred, option): + pred = pred.strip() + option_candidate = list(option.keys()) + if len(pred) == 1: + return pred + elif len(pred) != 1 and pred[0] in option_candidate: + return pred[0] + elif len(pred) != 1 and pred[0] not in option_candidate: + for k, v in option.items(): + if v in pred: + return k + + return pred + + +def evaluate_chat_model(): + prompt = "Answer with the option's letter from the given choices directly." + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = MMVPDataset( + root=ds_collections[ds_name]['root'], # hf dataset path. + prompt=prompt, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, answers, data_ids, options) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + if len(options[0]) == 0: + preds = [pred] + else: + preds = [post_process(pred, options[0])] + + for question, pred, answer, data_id in zip(questions, preds, answers, data_ids): + outputs.append({ + 'question': question, + 'answer': pred, + 'gt_answers': answer, + 'data_id': data_id + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.jsonl' + output_path = os.path.join(args.out_dir, results_file) + writer = open(output_path, 'w') + + num_correct, num_total = 0, 0 + index, round_correct = 0, 0 + for item in merged_outputs: + writer.write(json.dumps(item) + '\n') + answer = item['answer'] + gt_answer = item['gt_answers'] + + index += 1 + + if answer == gt_answer: + round_correct += 1 + if index == 2: + index = 0 + if round_correct == 2: + num_correct += 1 + round_correct = 0 + + num_total += 1 + + writer.close() + print('Results saved to {}'.format(output_path)) + print(f'The accuracy is {num_correct/num_total}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='MMVP') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/mvbench/evaluate_mvbench.py b/internvl_chat_audio/eval/mvbench/evaluate_mvbench.py new file mode 100644 index 00000000..fa85b669 --- /dev/null +++ b/internvl_chat_audio/eval/mvbench/evaluate_mvbench.py @@ -0,0 +1,405 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import cv2 +import imageio +import numpy as np +import torch +from decord import VideoReader, cpu +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from torch.utils.data import Dataset +from tqdm import tqdm + +data_list = { + 'Action Sequence': ('action_sequence.json', './data/MVBench/video/star/Charades_v1_480/', 'video', True), + # has start & end + 'Action Prediction': ('action_prediction.json', './data/MVBench/video/star/Charades_v1_480/', 'video', True), + # has start & end + 'Action Antonym': ('action_antonym.json', './data/MVBench/video/ssv2_video/', 'video', False), + 'Fine-grained Action': ( + 'fine_grained_action.json', './data/MVBench/video/Moments_in_Time_Raw/videos/', 'video', False), + 'Unexpected Action': ('unexpected_action.json', './data/MVBench/video/FunQA_test/test/', 'video', False), + 'Object Existence': ('object_existence.json', './data/MVBench/video/clevrer/video_validation/', 'video', False), + 'Object Interaction': ('object_interaction.json', './data/MVBench/video/star/Charades_v1_480/', 'video', True), + # has start & end + 'Object Shuffle': ('object_shuffle.json', './data/MVBench/video/perception/videos/', 'video', False), + 'Moving Direction': ('moving_direction.json', './data/MVBench/video/clevrer/video_validation/', 'video', False), + 'Action Localization': ('action_localization.json', './data/MVBench/video/sta/sta_video/', 'video', True), + # has start & end + 'Scene Transition': ('scene_transition.json', './data/MVBench/video/scene_qa/video/', 'video', False), + 'Action Count': ('action_count.json', './data/MVBench/video/perception/videos/', 'video', False), + 'Moving Count': ('moving_count.json', './data/MVBench/video/clevrer/video_validation/', 'video', False), + 'Moving Attribute': ('moving_attribute.json', './data/MVBench/video/clevrer/video_validation/', 'video', False), + 'State Change': ('state_change.json', './data/MVBench/video/perception/videos/', 'video', False), + 'Fine-grained Pose': ('fine_grained_pose.json', './data/MVBench/video/nturgbd/', 'video', False), + 'Character Order': ('character_order.json', './data/MVBench/video/perception/videos/', 'video', False), + 'Egocentric Navigation': ('egocentric_navigation.json', './data/MVBench/video/vlnqa/', 'video', False), + 'Episodic Reasoning': ('episodic_reasoning.json', './data/MVBench/video/tvqa/frames_fps3_hq/', 'frame', True), + # has start & end, read frame + 'Counterfactual Inference': ( + 'counterfactual_inference.json', './data/MVBench/video/clevrer/video_validation/', 'video', False), +} + +data_dir = './data/MVBench/json' + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + answers = [_['answer'] for _ in batches] + num_patches_lists = [_['num_patches_list'] for _ in batches] + task_types = [_['task_type'] for _ in batches] + return pixel_values, questions, answers, num_patches_lists, task_types + + +class MVBenchDataset(torch.utils.data.Dataset): + + def __init__(self, data_dir, data_list, prompt, question_prompt, num_segments=16, input_size=224, + dynamic_image_size=False, use_thumbnail=False, max_num=6): + self.data_list = [] + for k, v in data_list.items(): + with open(os.path.join(data_dir, v[0]), 'r') as f: + json_data = json.load(f) + for data in json_data: + self.data_list.append({ + 'task_type': k, + 'prefix': v[1], + 'data_type': v[2], + 'bound': v[3], + 'data': data + }) + self.decord_method = { + 'video': self.read_video, + 'gif': self.read_gif, + 'frame': self.read_frame, + } + self.prompt = prompt + self.question_prompt = question_prompt + self.input_size = input_size + self.num_segments = num_segments + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data_list) + + def __str__(self): + len_list = {} + option_list = {} + for data in self.data_list: + if data['task_type'] not in len_list: + len_list[data['task_type']] = 0 + len_list[data['task_type']] += 1 + if data['task_type'] not in option_list: + option_list[data['task_type']] = 0 + option_list[data['task_type']] += len(data['data']['candidates']) + + correct = 0 + total = 0 + res = f'There are {len(self.data_list)} videos as follow:\n' + for k, v in len_list.items(): + correct += len_list[k] + total += option_list[k] + res += f'{v} for {k} ({option_list[k]} options => {len_list[k] / option_list[k] * 100:.2f}%)\n' + correct = correct + 1 / option_list[k] + res += f'Total random accuracy: {correct / total * 100:.2f}%' + return res.rstrip() + + def get_index(self, bound, fps, max_frame, first_idx=0): + if bound: + start, end = bound[0], bound[1] + else: + start, end = -100000, 100000 + start_idx = max(first_idx, round(start * fps)) + end_idx = min(round(end * fps), max_frame) + seg_size = float(end_idx - start_idx) / self.num_segments + frame_indices = np.array([ + int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) + for idx in range(self.num_segments) + ]) + return frame_indices + + def read_video(self, video_path, bound=None): + vr = VideoReader(video_path, ctx=cpu(0), num_threads=1) + max_frame = len(vr) - 1 + fps = float(vr.get_avg_fps()) + + images_group = list() + frame_indices = self.get_index(bound, fps, max_frame, first_idx=0) + for frame_index in frame_indices: + img = Image.fromarray(vr[frame_index].asnumpy()) + images_group.append(img) + + return images_group + + def read_gif(self, video_path, bound=None, fps=25): + gif = imageio.get_reader(video_path) + max_frame = len(gif) - 1 + + images_group = list() + frame_indices = self.get_index(bound, fps, max_frame, first_idx=0) + for index, frame in enumerate(gif): + if index in frame_indices: + img = cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB) + img = Image.fromarray(img) + images_group.append(img) + + return images_group + + def read_frame(self, video_path, bound=None, fps=3): + max_frame = len(os.listdir(video_path)) + images_group = list() + frame_indices = self.get_index(bound, fps, max_frame, first_idx=1) # frame_idx starts from 1 + for frame_index in frame_indices: + img = Image.open(os.path.join(video_path, f'{frame_index:05d}.jpg')) + images_group.append(img) + + return images_group + + def qa_template(self, data): + question = f"Question: {data['question']}\n" + question += 'Options:\n' + answer = data['answer'] + answer_idx = -1 + for idx, c in enumerate(data['candidates']): + question += f"({chr(ord('A') + idx)}) {c}\n" + if c == answer: + answer_idx = idx + question = question.rstrip() + answer = f"({chr(ord('A') + answer_idx)}) {answer}" + return question, answer + + def __getitem__(self, idx): + decord_method = self.decord_method[self.data_list[idx]['data_type']] + bound = None + if self.data_list[idx]['bound']: + bound = ( + self.data_list[idx]['data']['start'], + self.data_list[idx]['data']['end'], + ) + video_path = os.path.join(self.data_list[idx]['prefix'], self.data_list[idx]['data']['video']) + image_list = decord_method(video_path, bound) + special_tokens = '\n'.join(['Frame{}: '.format(i + 1) for i in range(len(image_list))]) + + question, answer = self.qa_template(self.data_list[idx]['data']) + question = special_tokens + '\n' + self.prompt + '\n' + question + self.question_prompt + + raw_images = [] + num_patches_list = [] + pixel_values = [] + for image in image_list: + raw_images.append(image) + if self.dynamic_image_size: + patches = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + patches = [image] + num_patches_list.append(len(patches)) + pixel_values.extend([self.transform(patch) for patch in patches]) + + pixel_values = torch.stack(pixel_values) + + return { + 'question': question, + 'pixel_values': pixel_values, + 'answer': answer, + 'num_patches_list': num_patches_list, + 'task_type': self.data_list[idx]['task_type'] + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def check_ans(pred, gt): + flag = False + pred = pred.replace('Answer: ', '') + + pred_list = pred.lower().split(' ') + pred_option, pred_content = pred_list[0], ' '.join(pred_list[1:]) + gt_list = gt.lower().split(' ') + gt_option, gt_content = gt_list[0], ' '.join(gt_list[1:]) + if gt_content[-1] == '.': + gt_content = gt_content[:-1] + + if pred_option.replace('.', '') in gt_option: + flag = True + elif gt_option in pred_option: + flag = True + + return flag + + +def evaluate_chat_model(): + random.seed(args.seed) + prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = '\nOnly give the best option.' + + vid_dataset = MVBenchDataset( + data_dir, data_list, + prompt=prompt, + question_prompt=question_prompt, + num_segments=args.num_segments, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num) + dataloader = torch.utils.data.DataLoader( + dataset=vid_dataset, + sampler=InferenceSampler(len(vid_dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, answers, num_patches_lists, task_types) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=1000, + min_new_tokens=1, + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + num_patches_list=num_patches_lists[0], + question=questions[0], + generation_config=generation_config, + verbose=True + ) + outputs.append({ + 'question': questions[0], + 'pred': pred, + 'gt': answers[0], + 'task_type': task_types[0], + }) + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + + print(f'Evaluating MVBench ...') + correct, total, acc_dict = 0, 0, {} + for item in merged_outputs: + task_type = item['task_type'] + pred = item['pred'] + gt = item['gt'] + if task_type not in acc_dict: + acc_dict[task_type] = [0, 0] # correct, total + acc_dict[task_type][1] += 1 + total += 1 + + if check_ans(pred, gt): + acc_dict[task_type][0] += 1 + correct += 1 + + final_res = {} + for k, v in acc_dict.items(): + final_res[k] = v[0] / v[1] * 100 + final_res['Avg'] = correct / total * 100 + print(final_res) + + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'MVBench_{time_prefix}' + output_path = os.path.join(args.out_dir, results_file) + with open(f'{output_path}.json', 'w') as f: + json.dump(outputs, f) + with open(f'{output_path}_result_final.json', 'w') as f: + json.dump(final_res, f) + print('Results saved to {}'.format(output_path)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='mvbench') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=1) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + parser.add_argument('--num_segments', type=int, default=16) + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/pope/eval_pope.py b/internvl_chat_audio/eval/pope/eval_pope.py new file mode 100644 index 00000000..47594cf7 --- /dev/null +++ b/internvl_chat_audio/eval/pope/eval_pope.py @@ -0,0 +1,83 @@ +import argparse +import json +import os + + +def eval_pope(answers, label_file): + label_list = [json.loads(q)['label'] for q in open(label_file, 'r')] + + for answer in answers: + text = answer['text'] + + # Only keep the first sentence + if text.find('.') != -1: + text = text.split('.')[0] + + text = text.replace(',', '') + words = text.split(' ') + if 'No' in words or 'not' in words or 'no' in words: + answer['text'] = 'no' + else: + answer['text'] = 'yes' + + for i in range(len(label_list)): + if label_list[i] == 'no': + label_list[i] = 0 + else: + label_list[i] = 1 + + pred_list = [] + for answer in answers: + if answer['text'] == 'no': + pred_list.append(0) + else: + pred_list.append(1) + + pos = 1 + neg = 0 + yes_ratio = pred_list.count(1) / len(pred_list) + + TP, TN, FP, FN = 0, 0, 0, 0 + for pred, label in zip(pred_list, label_list): + if pred == pos and label == pos: + TP += 1 + elif pred == pos and label == neg: + FP += 1 + elif pred == neg and label == neg: + TN += 1 + elif pred == neg and label == pos: + FN += 1 + + print('TP\tFP\tTN\tFN\t') + print('{}\t{}\t{}\t{}'.format(TP, FP, TN, FN)) + + precision = float(TP) / float(TP + FP) + recall = float(TP) / float(TP + FN) + f1 = 2 * precision * recall / (precision + recall) + acc = (TP + TN) / (TP + TN + FP + FN) + print('Accuracy: {}'.format(acc)) + print('Precision: {}'.format(precision)) + print('Recall: {}'.format(recall)) + print('F1 score: {}'.format(f1)) + print('Yes ratio: {}'.format(yes_ratio)) + print('%.3f, %.3f, %.3f, %.3f, %.3f' % (f1, acc, precision, recall, yes_ratio)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--annotation-dir', type=str) + parser.add_argument('--question-file', type=str) + parser.add_argument('--result-file', type=str) + args = parser.parse_args() + + questions = [json.loads(line) for line in open(args.question_file)] + questions = {question['question_id']: question for question in questions} + answers = json.loads(open(args.result_file).read()) + for file in os.listdir(args.annotation_dir): + assert file.startswith('coco_pope_') + assert file.endswith('.json') + category = file[10:-5] + cur_answers = [x for x in answers if questions[x['question_id']]['category'] == category] + print('Category: {}, # samples: {}'.format(category, len(cur_answers))) + eval_pope(cur_answers, os.path.join(args.annotation_dir, file)) + print('====================================') diff --git a/internvl_chat_audio/eval/pope/evaluate_pope.py b/internvl_chat_audio/eval/pope/evaluate_pope.py new file mode 100644 index 00000000..b62bec5f --- /dev/null +++ b/internvl_chat_audio/eval/pope/evaluate_pope.py @@ -0,0 +1,223 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from tqdm import tqdm + +ds_collections = { + 'pope': { + 'root': 'data/pope/val2014', + 'question': 'data/pope/llava_pope_test.jsonl', + 'metric': None, + 'max_new_tokens': 100, + 'min_new_tokens': 1, + } +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + question_ids = [_['question_id'] for _ in batches] + annotations = [_['annotation'] for _ in batches] + + return pixel_values, questions, question_ids, annotations + + +class VQADataset(torch.utils.data.Dataset): + + def __init__(self, root, data, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + self.root = root + self.data = open(data).readlines() + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + data = json.loads(self.data[idx].strip()) + image, question, question_id, annotation = data['image'], data[ + 'text'], data['question_id'], data.get('answer', None) + + image = os.path.join(self.root, image) + image = Image.open(image).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + question = question + ' ' + self.prompt + return { + 'question_id': question_id, + 'question': question, + 'pixel_values': pixel_values, + 'annotation': annotation + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def evaluate_chat_model(): + prompt = 'Answer the question using a single word or phrase.' + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = VQADataset( + root=ds_collections[ds_name]['root'], + data=ds_collections[ds_name]['question'], + prompt='', + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, question_ids, annotations) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + answers = [pred] + + for question_id, answer, annotation in zip(question_ids, answers, annotations): + outputs.append({ + 'question_id': question_id, + 'text': pred, + 'model_id': args.checkpoint, + 'metadata': {}, + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + results_file = os.path.join(args.out_dir, results_file) + json.dump(merged_outputs, open(results_file, 'w')) + print('Results saved to {}'.format(results_file)) + cmd = 'python eval/pope/eval_pope.py ' \ + '--annotation-dir ./data/pope/coco ' \ + '--question-file ./data/pope/llava_pope_test.jsonl ' \ + '--result-file ' + results_file + print(cmd) + os.system(cmd) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='pope') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/refcoco/evaluate_grounding.py b/internvl_chat_audio/eval/refcoco/evaluate_grounding.py new file mode 100644 index 00000000..43f050b1 --- /dev/null +++ b/internvl_chat_audio/eval/refcoco/evaluate_grounding.py @@ -0,0 +1,276 @@ +import argparse +import itertools +import json +import os +import random +import re +import time +from functools import partial + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from torchvision.ops.boxes import box_area +from tqdm import tqdm + +ds_collections = { + 'refcoco_val': 'data/refcoco/refcoco_val.jsonl', + 'refcoco_testA': 'data/refcoco/refcoco_testA.jsonl', + 'refcoco_testB': 'data/refcoco/refcoco_testB.jsonl', + 'refcoco+_val': 'data/refcoco/refcoco+_val.jsonl', + 'refcoco+_testA': 'data/refcoco/refcoco+_testA.jsonl', + 'refcoco+_testB': 'data/refcoco/refcoco+_testB.jsonl', + 'refcocog_val': 'data/refcoco/refcocog_val.jsonl', + 'refcocog_test': 'data/refcoco/refcocog_test.jsonl', +} + + +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / union + return iou, union + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + texts = [_['text'] for _ in batches] + bboxes = [_['bbox'] for _ in batches] + hws = [_['hw'] for _ in batches] + return pixel_values, texts, bboxes, hws + + +class RefCOCODataset(torch.utils.data.Dataset): + + def __init__(self, test, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + self.datas = open(test).readlines() + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.datas) + + def __getitem__(self, idx): + data = json.loads(self.datas[idx].strip()) + image = data['image'] + text = data['sent'] + bbox = data['bbox'] + + w, h = data['width'], data['height'] + + image = Image.open(image).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + return { + 'text': self.prompt.format(text), + 'pixel_values': pixel_values, + 'bbox': bbox, + 'hw': (h, w), + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def evaluate_chat_model(): + print('prompt:', prompt) + random.seed(args.seed) + summaries = [] + + for ds_name in args.datasets: + dataset = RefCOCODataset( + test=ds_collections[ds_name], + prompt=prompt, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, bboxes, hws) in enumerate(tqdm(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=100, + min_new_tokens=1, + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + answers = [pred] + + for bbox, hw, answer in zip(bboxes, hws, answers): + outputs.append({ + 'answer': answer, + 'gt_bbox': bbox, + 'hw': hw, + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, outputs) + + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + results_file = os.path.join(args.out_dir, results_file) + json.dump(merged_outputs, open(results_file, 'w')) + + correct = total_cnt = 0 + for i, output in enumerate(merged_outputs): + predict_bbox = re.findall(PATTERN, output['answer']) + try: + predict_bbox = (float(predict_bbox[0][0]), float(predict_bbox[0][1]), float(predict_bbox[0][2]), + float(predict_bbox[0][3])) + except: + predict_bbox = (0., 0., 0., 0.) + target_bbox = torch.tensor(output['gt_bbox'], + dtype=torch.float32).view(-1, 4) + predict_bbox = torch.tensor(predict_bbox, + dtype=torch.float32).view(-1, 4) + if predict_bbox.sum() >= 4: + predict_bbox = predict_bbox / 1000 + predict_bbox[:, 0::2] *= output['hw'][1] + predict_bbox[:, 1::2] *= output['hw'][0] + iou, _ = box_iou(predict_bbox, target_bbox) + iou = iou.item() + total_cnt += 1 + if iou >= 0.5: + correct += 1 + + print(f'Evaluating {ds_name} ...') + print(f'Precision @ 1: {correct / total_cnt} \n') + summaries.append([args.checkpoint, ds_name, f'Precision @ 1: {correct / total_cnt} \n']) + + torch.distributed.barrier() + + out_path = '_'.join(args.checkpoint.split('/')[-2:]) + writer = open(os.path.join(args.out_dir, f'{out_path}.txt'), 'a') + print(f"write results to file {os.path.join(args.out_dir, f'{out_path}.txt')}") + for summary in summaries: + print(summary) + writer.write(f'{summary}\n') + writer.close() + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='refcoco_val,refcoco_testA,refcoco_testB,' + 'refcoco+_val,refcoco+_testA,refcoco+_testB,' + 'refcocog_val,refcocog_test') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--sample', type=bool, default=False) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + PATTERN = re.compile(r'\[*\[(.*?),(.*?),(.*?),(.*?)\]\]*') + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + prompt = 'Please provide the bounding box coordinate of the region this sentence describes: {}' + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + print(f'[test] max_num: {args.max_num}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/scienceqa/evaluate_scienceqa.py b/internvl_chat_audio/eval/scienceqa/evaluate_scienceqa.py new file mode 100644 index 00000000..a4f6aed9 --- /dev/null +++ b/internvl_chat_audio/eval/scienceqa/evaluate_scienceqa.py @@ -0,0 +1,255 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from torch.utils.data import Dataset +from tqdm import tqdm + +ds_collections = { + 'sqa_test': { + 'root': 'data/scienceqa/scienceqa_test_img.jsonl', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + }, +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + answers = [_['answer'] for _ in batches] + image_paths = [_['image_path'] for _ in batches] + options = [_['option'] for _ in batches] + return pixel_values, questions, answers, image_paths, options + + +class ScienceQADataset(torch.utils.data.Dataset): + + def __init__(self, root, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + f = open(root, 'r', encoding='utf-8') + self.data = [json.loads(line) for line in f.readlines()] + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + data = self.data[idx] + image_path = data['image'] + hint = data['hint'] if data['hint'] else None + question = data['question'] + + choices = data['choices'] + answer = data['answer'] + choice_list = [] + + options = {} + multiple_choices = ['A', 'B', 'C', 'D', 'E'] + for i, c in enumerate(choices): + choice_list.append('{}. {}'.format(multiple_choices[i], c)) + options[multiple_choices[i]] = c + choice_txt = '\n'.join(choice_list) + + image = Image.open(image_path).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + + if hint is not None: + question = hint + '\n' + question + question += '\n' + choice_txt + question += '\n' + self.prompt + + return { + 'question': question, + 'pixel_values': pixel_values, + 'answer': multiple_choices[answer], + 'image_path': image_path, + 'option': options + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def post_process(pred, option): + pred = pred.strip() + option_candidate = list(option.keys()) + if len(pred) == 1: + return pred + elif len(pred) != 1 and pred[0] in option_candidate: + return pred[0] + elif len(pred) != 1 and pred[0] not in option_candidate: + for k, v in option.items(): + if v in pred: + return k + + return pred + + +def evaluate_chat_model(): + prompt = "Answer with the option's letter from the given choices directly." + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = ScienceQADataset( + root=ds_collections[ds_name]['root'], + prompt=prompt, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, answers, image_paths, options) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + preds = [post_process(pred, options[0])] + + for question, pred, answer, image_path in zip(questions, preds, answers, image_paths): + outputs.append({ + 'question': question, + 'answer': pred, + 'gt_answers': answer, + 'image_path': image_path + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.jsonl' + output_path = os.path.join(args.out_dir, results_file) + with open(output_path, 'w') as f: + for output in merged_outputs: + f.write(json.dumps(output) + '\n') + print('Results saved to {}'.format(output_path)) + cnt = 0 + for item in merged_outputs: + if item['answer'] == item['gt_answers']: + cnt += 1 + print(f'Acc@1: {cnt / len(merged_outputs)}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='sqa_test') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/seed/calculation.py b/internvl_chat_audio/eval/seed/calculation.py new file mode 100644 index 00000000..e92d4e3f --- /dev/null +++ b/internvl_chat_audio/eval/seed/calculation.py @@ -0,0 +1,99 @@ +import argparse +import json +import os + +argparse = argparse.ArgumentParser() +argparse.add_argument('--image_result_file', type=str, default='') +argparse.add_argument('--anno_path', type=str, default='data/SEED/SEED-Bench.json') + +args = argparse.parse_args() +image_result_file = args.image_result_file +anno_path = args.anno_path + +assert image_result_file.endswith('.jsonl') + + +def is_integer_string(s): + try: + int(s) + return True + except ValueError: + return False + + +def filter_questions(data, task='all'): + if task == 'image': + return [q for q in data if 1 <= q['question_type_id'] <= 9] + elif task == 'video': + return [q for q in data if 10 <= q['question_type_id'] <= 12] + elif task == 'all': + return data + elif is_integer_string(task): + return [q for q in data if q['question_type_id'] == int(task)] + else: + raise ValueError(f'Invalid task: {task}') + + +if __name__ == '__main__': + + qa_anno = json.load(open(anno_path, 'rb')) + if 'questions' in qa_anno.keys(): + question_type = qa_anno['question_type'] + question_id_type = {v: k for k, v in question_type.items()} + qa_anno = qa_anno['questions'] + + qa_anno = filter_questions(qa_anno, 'all') + print(f'length: {len(qa_anno)}') + + with open(image_result_file, 'r') as f: + + image_result = [json.loads(line) for line in f.readlines()] + + results = [] + + results.extend(image_result) + + qa_id_anno = {} + for item in qa_anno: + question_id = str(item['question_id']) + qa_id_anno[question_id] = item + + type_counts = {k: [] for k, v in question_id_type.items()} + + for item in results: + pred, gt, question_id = item['prediction'], item['answer'], item['question_id'] + question_id = str(question_id) + question_type = qa_id_anno[question_id]['question_type_id'] + data_type = qa_id_anno[question_id]['data_type'] + gt = qa_id_anno[question_id]['answer'] + if len(pred) != 1: + pred = pred[0] + if pred == gt: + type_counts[question_type].append(1) + else: + type_counts[question_type].append(0) + + print('Accuracy for each data type:') + total_count, image_count, video_count = 0, 0, 0 + total_correct, image_correct, video_correct = 0, 0, 0 + for data_type_id, result in type_counts.items(): + accuracy = sum(result) / len(result) * 100 + data_type = question_id_type[data_type_id] + print(f'Data type {data_type}: {accuracy:.2f}%') + + total_count += len(result) + total_correct += sum(result) + if data_type_id >= 1 and data_type_id <= 9: + image_count += len(result) + image_correct += sum(result) + else: + video_count += len(result) + video_correct += sum(result) + + total_accuracy = total_correct / total_count * 100 + image_accuracy = image_correct / image_count * 100 + video_accuracy = video_correct / video_count * 100 + + print(f'Total accuracy: {total_accuracy:.2f}%') + print(f'Image accuracy: {image_accuracy:.2f}%') + print(f'Video accuracy: {video_accuracy:.2f}%') diff --git a/internvl_chat_audio/eval/seed/evaluate_seed.py b/internvl_chat_audio/eval/seed/evaluate_seed.py new file mode 100644 index 00000000..cc8a15fc --- /dev/null +++ b/internvl_chat_audio/eval/seed/evaluate_seed.py @@ -0,0 +1,242 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from torch.utils.data import Dataset +from tqdm import tqdm + +ds_collections = { + 'SEEDv1': { + 'root': 'data/SEED/', + 'annotation': 'data/SEED/seed.jsonl', + 'max_new_tokens': 100, + 'min_new_tokens': 1, + }, +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + answers = [_['answer'] for _ in batches] + indexes = [_['index'] for _ in batches] + return pixel_values, questions, answers, indexes + + +class MultipleChoiceDataset(torch.utils.data.Dataset): + + def __init__(self, root, annotation, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + f = open(annotation, 'r', encoding='utf-8') + self.data = [json.loads(line) for line in f.readlines()] + self.root = root + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + data = self.data[idx] + question = data['text'] + image_path = os.path.join(self.root, data['image']) + image = Image.open(image_path).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + answer = data['answer'] if 'answer' in data else None + return { + 'question': question, + 'pixel_values': pixel_values, + 'answer': answer, + 'index': data['question_id'], + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def post_process(pred, option): + pred = pred.strip() + option_candidate = list(option.keys()) + if len(pred) == 1: + return pred + elif len(pred) != 1 and pred[0] in option_candidate: + return pred[0] + elif len(pred) != 1 and pred[0] not in option_candidate: + for k, v in option.items(): + if v in pred: + return k + + return pred + + +def evaluate_chat_model(): + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = MultipleChoiceDataset( + root=ds_collections[ds_name]['root'], + annotation=ds_collections[ds_name]['annotation'], + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, answers, indexes) in enumerate(tqdm(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + preds = [pred] + + for question, pred, answer, index in zip(questions, preds, answers, indexes): + outputs.append({ + 'question_id': index, + 'question': question, + 'prediction': pred, + 'answer': answer, + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.jsonl' + output_path = os.path.join(args.out_dir, results_file) + writer = open(output_path, 'w') + + results = [] + for item in merged_outputs: + writer.write(json.dumps(item) + '\n') + answer = item['answer'] + prediction = item['prediction'] + if prediction == answer: + results.append(1) + else: + results.append(0) + writer.close() + print('Results saved to {}'.format(output_path)) + print(f'Acc@1: {sum(results) / len(results)}') + cmd = f'python eval/seed/calculation.py --image_result_file {output_path}' + os.system(cmd) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='SEEDv1') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/tiny_lvlm/calculate_score.py b/internvl_chat_audio/eval/tiny_lvlm/calculate_score.py new file mode 100644 index 00000000..e031c05e --- /dev/null +++ b/internvl_chat_audio/eval/tiny_lvlm/calculate_score.py @@ -0,0 +1,48 @@ +import argparse +import json + +try: + from .tools import VQAEval +except: + from tools import VQAEval + + +def parse_args(): + parser = argparse.ArgumentParser(description='Demo') + parser.add_argument('--file-path', type=str, default='results/updated_datasets_231221114523.json') + args = parser.parse_args() + return args + + +def main(args): + data = json.loads(open(args.file_path).read()) + overall_score = 0 + results = {} + dataset_names = ['Visual_Reasoning', 'Visual_Perception', 'Visual_Knowledge_Acquisition', + 'Visual_Commonsense', 'Object_Hallucination'] + for item in data: + task_type = item['image_path'].split('/')[-2] + assert task_type in dataset_names + if task_type in results: + results[task_type].append(item) + else: + results[task_type] = [item] + + for k, v in results.items(): + eval = VQAEval() + correct = 0 + num = 0 + for i in range(len(v)): + gt_answers = v[i]['gt_answers'] + answer = v[i]['answer'] + if eval.evaluate(answer, gt_answers) == 1: + correct += 1 + num += 1 + overall_score += float(correct) / num + print(f'{k}: {float(correct) / num}') + print(f'Overall: {overall_score}') + + +if __name__ == '__main__': + args = parse_args() + main(args) diff --git a/internvl_chat_audio/eval/tiny_lvlm/evaluate_lvlm.py b/internvl_chat_audio/eval/tiny_lvlm/evaluate_lvlm.py new file mode 100644 index 00000000..05de28fd --- /dev/null +++ b/internvl_chat_audio/eval/tiny_lvlm/evaluate_lvlm.py @@ -0,0 +1,230 @@ +import argparse +import itertools +import json +import os +import random +import time +from functools import partial + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from tqdm import tqdm + +ds_collections = { + 'updated_datasets': { + 'root': 'data/tiny_lvlm/updated_datasets/', + 'max_new_tokens': 30, + 'min_new_tokens': 1, + } +} + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + annotations = [_['annotation'] for _ in batches] + image_paths = [_['image_path'] for _ in batches] + + return pixel_values, questions, annotations, image_paths + + +class VQADataset(torch.utils.data.Dataset): + + def __init__(self, root, prompt, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + dirnames = [os.path.join(root, item) for item in os.listdir(root)] + dirnames = [item for item in dirnames if os.path.exists(os.path.join(item, 'dataset.json'))] + sorted(dirnames) + + self.roots = [] + self.items = [] + for item in dirnames: + data_path = os.path.join(item, 'dataset.json') + data = json.loads(open(data_path).read()) + for data_line in data: + self.roots.append(item) + self.items.append(data_line) + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.max_num = max_num + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.items) + + def __getitem__(self, idx): + root = self.roots[idx] + item = self.items[idx] + image_path, question, annotation = item['image_path'], item['question'], item['gt_answers'] + image_path = os.path.join(root, image_path) + image = Image.open(image_path).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + question = question + ' ' + self.prompt + return { + 'question': question, + 'pixel_values': pixel_values, + 'annotation': annotation, + 'image_path': image_path, + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def evaluate_chat_model(): + prompt = 'Answer the question using a single word or phrase.' + random.seed(args.seed) + + for ds_name in args.datasets: + dataset = VQADataset( + root=ds_collections[ds_name]['root'], + prompt=prompt, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, annotations, image_paths) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=ds_collections[ds_name]['min_new_tokens'], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + answers = [pred] + + for question, answer, annotation, image_path in zip(questions, answers, annotations, image_paths): + task_type = image_path.split('/')[-2] + outputs.append({ + 'question': question, + 'answer': answer, + 'gt_answers': annotation, + 'image_path': image_path, + 'task_type': task_type + }) + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + results_file = os.path.join(args.out_dir, results_file) + json.dump(merged_outputs, open(results_file, 'w')) + print('Results saved to {}'.format(results_file)) + cmd = 'python eval/tiny_lvlm/calculate_score.py ' \ + '--file-path ' + results_file + print(cmd) + os.system(cmd) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, default='updated_datasets') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/tiny_lvlm/tools.py b/internvl_chat_audio/eval/tiny_lvlm/tools.py new file mode 100644 index 00000000..7942569a --- /dev/null +++ b/internvl_chat_audio/eval/tiny_lvlm/tools.py @@ -0,0 +1,256 @@ +# https://github.com/OpenGVLab/Multi-Modality-Arena/blob/main/tiny_lvlm_evaluation/utils/tools.py +import re + + +def remove_special_chars(s): + pattern = r'[^a-zA-Z0-9\s]' + s = re.sub(pattern, '', s) + return s + + +def has_word(sentence, word): + pattern = r'\b' + re.escape(word) + r'\b' + match = re.search(pattern, sentence) + if match: + return True + else: + return False + + +class VQAEval: + def __init__(self): + self.contractions = { + 'aint': "ain't", + 'arent': "aren't", + 'cant': "can't", + 'couldve': "could've", + 'couldnt': "couldn't", + "couldn'tve": "couldn't've", + "couldnt've": "couldn't've", + 'didnt': "didn't", + 'doesnt': "doesn't", + 'dont': "don't", + 'hadnt': "hadn't", + "hadnt've": "hadn't've", + "hadn'tve": "hadn't've", + 'hasnt': "hasn't", + 'havent': "haven't", + 'hed': "he'd", + "hed've": "he'd've", + "he'dve": "he'd've", + 'hes': "he's", + 'howd': "how'd", + 'howll': "how'll", + 'hows': "how's", + "Id've": "I'd've", + "I'dve": "I'd've", + 'Im': "I'm", + 'Ive': "I've", + 'isnt': "isn't", + 'itd': "it'd", + "itd've": "it'd've", + "it'dve": "it'd've", + 'itll': "it'll", + "let's": "let's", + 'maam': "ma'am", + 'mightnt': "mightn't", + "mightnt've": "mightn't've", + "mightn'tve": "mightn't've", + 'mightve': "might've", + 'mustnt': "mustn't", + 'mustve': "must've", + 'neednt': "needn't", + 'notve': "not've", + 'oclock': "o'clock", + 'oughtnt': "oughtn't", + "ow's'at": "'ow's'at", + "'ows'at": "'ow's'at", + "'ow'sat": "'ow's'at", + 'shant': "shan't", + "shed've": "she'd've", + "she'dve": "she'd've", + "she's": "she's", + 'shouldve': "should've", + 'shouldnt': "shouldn't", + "shouldnt've": "shouldn't've", + "shouldn'tve": "shouldn't've", + "somebody'd": 'somebodyd', + "somebodyd've": "somebody'd've", + "somebody'dve": "somebody'd've", + 'somebodyll': "somebody'll", + 'somebodys': "somebody's", + 'someoned': "someone'd", + "someoned've": "someone'd've", + "someone'dve": "someone'd've", + 'someonell': "someone'll", + 'someones': "someone's", + 'somethingd': "something'd", + "somethingd've": "something'd've", + "something'dve": "something'd've", + 'somethingll': "something'll", + 'thats': "that's", + 'thered': "there'd", + "thered've": "there'd've", + "there'dve": "there'd've", + 'therere': "there're", + 'theres': "there's", + 'theyd': "they'd", + "theyd've": "they'd've", + "they'dve": "they'd've", + 'theyll': "they'll", + 'theyre': "they're", + 'theyve': "they've", + 'twas': "'twas", + 'wasnt': "wasn't", + "wed've": "we'd've", + "we'dve": "we'd've", + 'weve': "we've", + 'werent': "weren't", + 'whatll': "what'll", + 'whatre': "what're", + 'whats': "what's", + 'whatve': "what've", + 'whens': "when's", + 'whered': "where'd", + 'wheres': "where's", + 'whereve': "where've", + 'whod': "who'd", + "whod've": "who'd've", + "who'dve": "who'd've", + 'wholl': "who'll", + 'whos': "who's", + 'whove': "who've", + 'whyll': "why'll", + 'whyre': "why're", + 'whys': "why's", + 'wont': "won't", + 'wouldve': "would've", + 'wouldnt': "wouldn't", + "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", + 'yall': "y'all", + "yall'll": "y'all'll", + "y'allll": "y'all'll", + "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", + "y'all'dve": "y'all'd've", + 'youd': "you'd", + "youd've": "you'd've", + "you'dve": "you'd've", + 'youll': "you'll", + 'youre': "you're", + 'youve': "you've", + } + self.manualMap = { + 'none': '0', + 'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10', + } + self.articles = ['a', 'an', 'the'] + + self.periodStrip = re.compile('(?!<=\d)(\.)(?!\d)') + self.commaStrip = re.compile('(\d)(\,)(\d)') + self.punct = [ + ';', + r'/', + '[', + ']', + '"', + '{', + '}', + '(', + ')', + '=', + '+', + '\\', + '_', + '-', + '>', + '<', + '@', + '`', + ',', + '?', + '!', + ] + + def evaluate(self, answer, gt_answers): + answer = answer.replace('\n', ' ') + answer = answer.replace('\t', ' ') + answer = answer.strip() + answer = self.processPunctuation(answer) + answer = self.processDigitArticle(answer) + if type(gt_answers) == list: + for i in range(len(gt_answers)): + gt_answers[i] = gt_answers[i].replace('\n', ' ') + gt_answers[i] = gt_answers[i].replace('\t', ' ') + gt_answers[i] = gt_answers[i].strip() + gt_answers[i] = self.processPunctuation(gt_answers[i]) + gt_answers[i] = self.processDigitArticle(gt_answers[i]) + if has_word(answer, gt_answers[i]): + return 1 + return 0 + else: + gt_answers = gt_answers.replace('\n', ' ') + gt_answers = gt_answers.replace('\t', ' ') + gt_answers = gt_answers.strip() + gt_answers = self.processPunctuation(gt_answers) + gt_answers = self.processDigitArticle(gt_answers) + if has_word(answer, gt_answers): + return 1 + else: + return 0 + + def evaluate_MRR(self, answer, gt_answers): + answer = answer.replace('\n', ' ') + answer = answer.replace('\t', ' ') + answer = answer.strip() + answer = self.processPunctuation(answer) + answer = self.processDigitArticle(answer) + if type(gt_answers) is str: + gt_answers = [gt_answers] + for i in range(len(gt_answers)): + gt_answers[i] = gt_answers[i].replace('\n', ' ') + gt_answers[i] = gt_answers[i].replace('\t', ' ') + gt_answers[i] = gt_answers[i].strip() + gt_answers[i] = self.processPunctuation(gt_answers[i]) + gt_answers[i] = self.processDigitArticle(gt_answers[i]) + if has_word(answer, gt_answers[i]): + return 1 / (i + 1) + return 0.0 + + def processPunctuation(self, inText): + outText = inText + for p in self.punct: + if (p + ' ' in inText or ' ' + p in inText) or ( + re.search(self.commaStrip, inText) is not None + ): + outText = outText.replace(p, '') + else: + outText = outText.replace(p, ' ') + outText = self.periodStrip.sub('', outText, re.UNICODE) + return outText + + def processDigitArticle(self, inText): + outText = [] + tempText = inText.lower().split() + for word in tempText: + word = self.manualMap.setdefault(word, word) + if word not in self.articles: + outText.append(word) + else: + pass + for wordId, word in enumerate(outText): + if word in self.contractions: + outText[wordId] = self.contractions[word] + outText = ' '.join(outText) + return outText diff --git a/internvl_chat_audio/eval/vqa/convert_gqa_for_eval.py b/internvl_chat_audio/eval/vqa/convert_gqa_for_eval.py new file mode 100644 index 00000000..407ad6d6 --- /dev/null +++ b/internvl_chat_audio/eval/vqa/convert_gqa_for_eval.py @@ -0,0 +1,17 @@ +import argparse +import json + +parser = argparse.ArgumentParser() +parser.add_argument('--src', type=str) +parser.add_argument('--dst', type=str) +args = parser.parse_args() + +all_answers = [] +data = json.load(open(args.src)) +for res in data: + question_id = res['questionId'] + answer = res['answer'].rstrip('.').lower() + all_answers.append({'questionId': question_id, 'prediction': answer}) + +with open(args.dst, 'w') as f: + json.dump(all_answers, f) diff --git a/internvl_chat_audio/eval/vqa/evaluate_vqa.py b/internvl_chat_audio/eval/vqa/evaluate_vqa.py new file mode 100644 index 00000000..a99b1e5f --- /dev/null +++ b/internvl_chat_audio/eval/vqa/evaluate_vqa.py @@ -0,0 +1,541 @@ +import argparse +import itertools +import json +import os +import random +import subprocess +import time +from functools import partial +from typing import Optional + +import torch +from internvl.model import load_model_and_tokenizer +from internvl.train.dataset import build_transform, dynamic_preprocess +from PIL import Image +from textvqa_eval import TextVQAAccuracyEvaluator +from tqdm import tqdm + +ds_collections = { + 'vqav2_val': { + 'train': 'data/vqav2/vqav2_train.jsonl', + 'test': 'data/vqav2/vqav2_val.jsonl', + 'question': 'data/vqav2/v2_OpenEnded_mscoco_val2014_questions.json', + 'annotation': 'data/vqav2/v2_mscoco_val2014_annotations.json', + 'metric': 'vqa_score', + 'max_new_tokens': 10, + }, + 'vqav2_testdev': { + 'train': 'data/vqav2/vqav2_train.jsonl', + 'test': 'data/vqav2/vqav2_testdev.jsonl', + 'metric': None, + 'max_new_tokens': 10, + }, + 'okvqa_val': { + 'train': 'data/okvqa/okvqa_train.jsonl', + 'test': 'data/okvqa/okvqa_val.jsonl', + 'question': 'data/okvqa/OpenEnded_mscoco_val2014_questions.json', + 'annotation': 'data/okvqa/mscoco_val2014_annotations.json', + 'metric': 'vqa_score', + 'max_new_tokens': 10, + }, + 'textvqa_val': { + 'train': 'data/textvqa/textvqa_train.jsonl', + 'test': 'data/textvqa/textvqa_val.jsonl', + 'question': 'data/textvqa/textvqa_val_questions.json', + 'annotation': 'data/textvqa/textvqa_val_annotations.json', + 'metric': 'vqa_score', + 'max_new_tokens': 10, + }, + 'textvqa_val_ocr': { + 'train': 'data/textvqa/textvqa_train.jsonl', + 'test': 'data/textvqa/textvqa_val_llava.jsonl', + 'question': 'data/textvqa/textvqa_val_questions.json', + 'annotation': 'data/textvqa/textvqa_val_annotations.json', + 'metric': 'vqa_score', + 'max_new_tokens': 10, + }, + 'vizwiz_val': { + 'train': 'data/vizwiz/vizwiz_train.jsonl', + 'test': 'data/vizwiz/vizwiz_val.jsonl', + 'question': 'data/vizwiz/vizwiz_val_questions.json', + 'annotation': 'data/vizwiz/vizwiz_val_annotations.json', + 'metric': 'vqa_score', + 'max_new_tokens': 10, + }, + 'vizwiz_test': { + 'train': 'data/vizwiz/vizwiz_train.jsonl', + 'test': 'data/vizwiz/vizwiz_test.jsonl', + 'metric': None, + 'max_new_tokens': 10, + }, + 'docvqa_val': { + 'train': 'data/docvqa/train.jsonl', + 'test': 'data/docvqa/val.jsonl', + 'annotation': 'data/docvqa/val/val_v1.0.json', + 'metric': 'anls', + 'max_new_tokens': 100, + }, + 'docvqa_test': { + 'train': 'data/docvqa/train.jsonl', + 'test': 'data/docvqa/test.jsonl', + 'metric': None, + 'max_new_tokens': 100, + }, + 'chartqa_test_human': { + 'train': 'data/chartqa/train_human.jsonl', + 'test': 'data/chartqa/test_human.jsonl', + 'metric': 'relaxed_accuracy', + 'max_new_tokens': 100, + }, + 'chartqa_test_augmented': { + 'train': 'data/chartqa/train_augmented.jsonl', + 'test': 'data/chartqa/test_augmented.jsonl', + 'metric': 'relaxed_accuracy', + 'max_new_tokens': 100, + }, + 'gqa_testdev': { + 'train': 'data/gqa/train.jsonl', + 'test': 'data/gqa/test_balanced.jsonl', + 'metric': 'accuracy', + 'max_new_tokens': 10, + }, + 'gqa_testdev_llava': { + 'train': 'data/gqa/train.jsonl', + 'test': 'data/gqa/llava_gqa_testdev_balanced_qwen_format.jsonl', + 'metric': 'accuracy', + 'max_new_tokens': 10, + }, + 'ocrvqa_val': { + 'train': 'data/ocrvqa/ocrvqa_train.jsonl', + 'test': 'data/ocrvqa/ocrvqa_val.jsonl', + 'metric': 'accuracy', + 'max_new_tokens': 100, + }, + 'ocrvqa_test': { + 'train': 'data/ocrvqa/ocrvqa_train.jsonl', + 'test': 'data/ocrvqa/ocrvqa_test.jsonl', + 'metric': 'accuracy', + 'max_new_tokens': 100, + }, + 'ai2diagram_test': { + 'train': 'data/ai2diagram/train.jsonl', + 'test': 'data/ai2diagram/test_vlmevalkit.jsonl', + 'metric': 'accuracy', + 'max_new_tokens': 10, + }, + 'infographicsvqa_val': { + 'train': 'data/infographicsvqa/train.jsonl', + 'test': 'data/infographicsvqa/val.jsonl', + 'annotation': 'data/infographicsvqa/infographicsVQA_val_v1.0_withQT.json', + 'metric': 'anls', + 'max_new_tokens': 100, + }, + 'infographicsvqa_test': { + 'train': 'data/infographicsvqa/train.jsonl', + 'test': 'data/infographicsvqa/test.jsonl', + 'annotation': 'data/infographicsvqa/infographicsVQA_test_v1.0.json', + 'metric': None, + 'max_new_tokens': 100, + } +} + + +# https://github.com/google-research/pix2struct/blob/main/pix2struct/metrics.py#L81 +def relaxed_correctness(target: str, + prediction: str, + max_relative_change: float = 0.05) -> bool: + """Calculates relaxed correctness. + + The correctness tolerates certain error ratio defined by max_relative_change. + See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1: + “Following Methani et al. (2020), we use a relaxed accuracy measure for the + numeric answers to allow a minor inaccuracy that may result from the automatic + data extraction process. We consider an answer to be correct if it is within + 5% of the gold answer. For non-numeric answers, we still need an exact match + to consider an answer to be correct.” + + Args: + target: Target string. + prediction: Predicted string. + max_relative_change: Maximum relative change. + + Returns: + Whether the prediction was correct given the specified tolerance. + """ + + def _to_float(text: str) -> Optional[float]: + try: + if text.endswith('%'): + # Convert percentages to floats. + return float(text.rstrip('%')) / 100.0 + else: + return float(text) + except ValueError: + return None + + prediction_float = _to_float(prediction) + target_float = _to_float(target) + if prediction_float is not None and target_float: + relative_change = abs(prediction_float - + target_float) / abs(target_float) + return relative_change <= max_relative_change + else: + return prediction.lower() == target.lower() + + +def evaluate_relaxed_accuracy(entries): + scores = [] + for elem in entries: + if isinstance(elem['annotation'], str): + elem['annotation'] = [elem['annotation']] + score = max([ + relaxed_correctness(elem['answer'].strip(), ann) + for ann in elem['annotation'] + ]) + scores.append(score) + return sum(scores) / len(scores) + + +def evaluate_exact_match_accuracy(entries): + scores = [] + for elem in entries: + if isinstance(elem['annotation'], str): + elem['annotation'] = [elem['annotation']] + score = max([ + (1.0 if + (elem['answer'].strip().lower() == ann.strip().lower()) else 0.0) + for ann in elem['annotation'] + ]) + scores.append(score) + return sum(scores) / len(scores) + + +def collate_fn(batches, tokenizer): + pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0) + questions = [_['question'] for _ in batches] + question_ids = [_['question_id'] for _ in batches] + annotations = [_['annotation'] for _ in batches] + + return pixel_values, questions, question_ids, annotations + + +class VQADataset(torch.utils.data.Dataset): + + def __init__(self, train, test, prompt, few_shot, input_size=224, dynamic_image_size=False, + use_thumbnail=False, max_num=6): + self.test = open(test).readlines() + self.prompt = prompt + self.input_size = input_size + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.few_shot = few_shot + self.max_num = max_num + if few_shot > 0: + self.train = open(train).readlines() + self.transform = build_transform(is_train=False, input_size=input_size) + + def __len__(self): + return len(self.test) + + def __getitem__(self, idx): + data = json.loads(self.test[idx].strip()) + image, question, question_id, annotation = data['image'], data[ + 'question'], data['question_id'], data.get('answer', None) + + few_shot_prompt = '' + if self.few_shot > 0: + few_shot_samples = random.sample(self.train, self.few_shot) + for sample in few_shot_samples: + sample = json.loads(sample.strip()) + few_shot_prompt += self.prompt.format( + sample['image'], + sample['question']) + f" {sample['answer']}" + + image = Image.open(image).convert('RGB') + if self.dynamic_image_size: + images = dynamic_preprocess(image, image_size=self.input_size, + use_thumbnail=self.use_thumbnail, + max_num=self.max_num) + else: + images = [image] + pixel_values = [self.transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + if len(self.prompt) != 0: + question = question + ' ' + self.prompt + return { + 'question_id': question_id, + 'question': question, + 'pixel_values': pixel_values, + 'annotation': annotation + } + + +class InferenceSampler(torch.utils.data.sampler.Sampler): + + def __init__(self, size): + self._size = int(size) + assert size > 0 + self._rank = torch.distributed.get_rank() + self._world_size = torch.distributed.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[:rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) + + +def post_process(response): + response = response.strip().split('.')[0].split( + ',')[0].split('!')[0].lower() + if 'is ' in response: + response = response.split('is ')[1] + if 'are ' in response: + response = response.split('are ')[1] + if 'a ' in response: + response = response.split('a ')[1] + if 'an ' in response: + response = response.split('an ')[1] + if 'the ' in response: + response = response.split('the ')[1] + if ' of' in response: + response = response.split(' of')[0] + response = response.strip() + return response + + +def evaluate_chat_model(): + base_prompt = 'Answer the question using a single word or phrase.' + vizwiz_prompt = "When the provided information is insufficient, respond with 'Unanswerable'. " + # infovqa_prompt = 'Answer the question directly.' + infovqa_prompt = 'Answer the question using a single word or phrase.' + ai2d_prompt = '' + random.seed(args.seed) + summaries = [] + + for ds_name in args.datasets: + if 'vizwiz' in ds_name: + input_prompt = vizwiz_prompt + base_prompt + elif 'ai2d' in ds_name: + input_prompt = ai2d_prompt + elif 'infographicsvqa' in ds_name: + input_prompt = infovqa_prompt + else: + input_prompt = base_prompt + + dataset = VQADataset( + train=ds_collections[ds_name]['train'], + test=ds_collections[ds_name]['test'], + prompt=input_prompt, + few_shot=args.few_shot, + input_size=image_size, + dynamic_image_size=args.dynamic, + use_thumbnail=use_thumbnail, + max_num=args.max_num + ) + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + sampler=InferenceSampler(len(dataset)), + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + collate_fn=partial(collate_fn, tokenizer=tokenizer), + ) + + outputs = [] + for _, (pixel_values, questions, question_ids, annotations) in tqdm(enumerate(dataloader)): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + generation_config = dict( + num_beams=args.num_beams, + max_new_tokens=ds_collections[ds_name]['max_new_tokens'], + min_new_tokens=1, + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + ) + pred = model.chat( + tokenizer=tokenizer, + pixel_values=pixel_values, + question=questions[0], + generation_config=generation_config, + verbose=True + ) + answers = [pred] + + for question, question_id, answer, annotation in zip(questions, question_ids, answers, annotations): + if ds_name in ['vqav2_val', 'vqav2_testdev', 'okvqa_val', 'textvqa_val', + 'vizwiz_val', 'textvqa_val_ocr']: + outputs.append({ + 'question': question, + 'question_id': question_id, + 'answer': answer, + }) + elif ds_name in ['docvqa_val', 'infographicsvqa_val', 'gqa_testdev', 'ocrvqa_val', + 'ocrvqa_test', 'gqa_testdev_llava', 'infographicsvqa_test',]: + outputs.append({ + 'question': question, + 'questionId': question_id, + 'answer': answer, + 'annotation': annotation, + }) + elif ds_name in ['ai2diagram_test']: + outputs.append({ + 'question': question, + 'image': question_id, + 'answer': answer, + 'annotation': annotation, + }) + elif ds_name in ['chartqa_test_human', 'chartqa_test_augmented']: + outputs.append({ + 'question': question, + 'answer': answer, + 'annotation': annotation, + }) + elif ds_name in ['docvqa_test']: + outputs.append({ + 'questionId': question_id, + 'answer': answer, + }) + elif ds_name in ['vizwiz_test']: + outputs.append({ + 'image': question_id.replace('data/vizwiz/test/', ''), + 'answer': answer, + }) + else: + raise NotImplementedError + + torch.distributed.barrier() + + world_size = torch.distributed.get_world_size() + merged_outputs = [None for _ in range(world_size)] + torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs)) + + merged_outputs = [json.loads(_) for _ in merged_outputs] + merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)] + + if torch.distributed.get_rank() == 0: + print(f'Evaluating {ds_name} ...') + time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) + results_file = f'{ds_name}_{time_prefix}.json' + results_file = os.path.join(args.out_dir, results_file) + json.dump(merged_outputs, open(results_file, 'w')) + print('Results saved to {}'.format(results_file)) + + if ds_collections[ds_name]['metric'] == 'vqa_score': + evaluator = TextVQAAccuracyEvaluator() + annotation = json.load(open(ds_collections[ds_name]['annotation'], 'r'))['annotations'] + question_id2answers = {} + for item in annotation: + question_id = item['question_id'] + answers = [answer['answer'] for answer in item['answers']] + question_id2answers[question_id] = answers + for item in merged_outputs: + item['pred_answer'] = item['answer'] + item['gt_answers'] = question_id2answers[item['question_id']] + accuracy = evaluator.eval_pred_list(merged_outputs) + print(ds_name, accuracy) + summaries.append([args.checkpoint, ds_name, accuracy]) + + elif ds_collections[ds_name]['metric'] == 'anls': + json.dump(merged_outputs, + open(results_file, 'w'), + ensure_ascii=False) + print('python eval/vqa/infographicsvqa_eval.py -g ' + + ds_collections[ds_name]['annotation'] + ' -s ' + + results_file) + os.system('python eval/vqa/infographicsvqa_eval.py -g ' + + ds_collections[ds_name]['annotation'] + ' -s ' + + results_file) + elif ds_collections[ds_name]['metric'] == 'relaxed_accuracy': + relaxed_accuracy = evaluate_relaxed_accuracy(merged_outputs) + print(ds_name, {'relaxed_accuracy': relaxed_accuracy}) + summaries.append([ds_name, {'relaxed_accuracy': relaxed_accuracy}]) + elif ds_collections[ds_name]['metric'] == 'accuracy': + if 'gqa' in ds_name: + dst_file = './data/gqa/testdev_balanced_predictions.json' + print('python eval/vqa/convert_gqa_for_eval.py --src ' + + results_file + ' --dst ' + dst_file) + python_path = 'python' + os.system(python_path + ' eval/vqa/convert_gqa_for_eval.py --src ' + + results_file + ' --dst ' + dst_file) + command = f'cd ./data/gqa/ && {python_path} eval.py --tier testdev_balanced && cd ../../' + print(command) + accuracy = subprocess.check_output(command, shell=True, universal_newlines=True) + else: + accuracy = {'accuracy': evaluate_exact_match_accuracy(merged_outputs)} + print(ds_name, accuracy) + summaries.append([args.checkpoint, ds_name, accuracy]) + + torch.distributed.barrier() + + out_path = '_'.join(args.checkpoint.split('/')[-2:]) + writer = open(os.path.join(args.out_dir, f'{out_path}.txt'), 'a') + print(f"write results to file {os.path.join(args.out_dir, f'{out_path}.txt')}") + for summary in summaries: + print(summary) + writer.write(f'{summary}\n') + writer.close() + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str, default='') + parser.add_argument('--datasets', type=str, + default='okvqa_val,textvqa_val,vizwiz_val,ai2diagram_test,gqa_testdev_llava') + parser.add_argument('--batch-size', type=int, default=1) + parser.add_argument('--num-workers', type=int, default=1) + parser.add_argument('--num-beams', type=int, default=5) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--out-dir', type=str, default='results') + parser.add_argument('--few-shot', type=int, default=0) + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--dynamic', action='store_true') + parser.add_argument('--max-num', type=int, default=6) + parser.add_argument('--load-in-8bit', action='store_true') + parser.add_argument('--load-in-4bit', action='store_true') + parser.add_argument('--auto', action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + args.datasets = args.datasets.split(',') + print('datasets:', args.datasets) + assert args.batch_size == 1, 'Only batch size 1 is supported' + + torch.distributed.init_process_group( + backend='nccl', + world_size=int(os.getenv('WORLD_SIZE', '1')), + rank=int(os.getenv('RANK', '0')), + ) + + torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0))) + + model, tokenizer = load_model_and_tokenizer(args) + image_size = model.config.force_image_size or model.config.vision_config.image_size + use_thumbnail = model.config.use_thumbnail + + total_params = sum(p.numel() for p in model.parameters()) / 1e9 + if total_params > 20 or args.dynamic: + args.num_beams = 1 + print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}') + else: + print(f'[test] total_params: {total_params}B') + print(f'[test] image_size: {image_size}') + print(f'[test] template: {model.config.template}') + print(f'[test] dynamic_image_size: {args.dynamic}') + print(f'[test] use_thumbnail: {use_thumbnail}') + + evaluate_chat_model() diff --git a/internvl_chat_audio/eval/vqa/infographicsvqa_eval.py b/internvl_chat_audio/eval/vqa/infographicsvqa_eval.py new file mode 100644 index 00000000..a312ca2d --- /dev/null +++ b/internvl_chat_audio/eval/vqa/infographicsvqa_eval.py @@ -0,0 +1,254 @@ +# This file can be downloaded from: https://www.docvqa.org/datasets/infographicvqa and https://rrc.cvc.uab.es/?ch=17&com=introduction + +import argparse +import json +import os + +question_ids_to_exclude = [] + +# answer_types = {'image span': 'Image-Span', 'question span': 'Question-Span', 'multiple spans': 'Multi-Span', 'non span': 'None span', 'list': 'List'} +answer_types = {'image span': 'Image-Span', 'question span': 'Question-Span', 'multiple spans': 'Multi-Span', + 'non span': 'None span'} +evidence_types = {'table/list': 'Table/list', 'textual': 'Text', 'photo/pciture/visual_objects': 'Visual/Layout', + 'figure': 'Figure', 'map': 'Map'} +reasoning_requirements = {'comparison': 'Sorting', 'arithmetic': 'Arithmetic', 'counting': 'Counting'} + + +def save_json(file_path, data): + with open(file_path, 'w+') as json_file: + json.dump(data, json_file) + + +def levenshtein_distance(s1, s2): + if len(s1) > len(s2): + s1, s2 = s2, s1 + + distances = range(len(s1) + 1) + for i2, c2 in enumerate(s2): + distances_ = [i2 + 1] + for i1, c1 in enumerate(s1): + if c1 == c2: + distances_.append(distances[i1]) + else: + distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1]))) + distances = distances_ + return distances[-1] + + +def validate_data(gtFilePath, submFilePath): + """ + Method validate_data: validates that all files in the results folder are correct (have the correct name contents). + Validates also that there are no missing files in the folder. + If some error detected, the method raises the error + """ + + gtJson = json.load(open(gtFilePath, 'rb')) + submJson = json.load(open(submFilePath, 'rb')) + + if 'data' not in gtJson: + raise Exception('The GT file is not valid (no data key)') + + if 'dataset_name' not in gtJson: + raise Exception('The GT file is not valid (no dataset_name key)') + + if isinstance(submJson, list) is False: + raise Exception('The Det file is not valid (root item must be an array)') + + if len(submJson) != len(gtJson['data']): + raise Exception('The Det file is not valid (invalid number of answers. Expected:' + str( + len(gtJson['data'])) + ' Found:' + str(len(submJson)) + ')') + + gtQuestions = sorted([r['questionId'] for r in gtJson['data']]) + res_id_to_index = {int(r['questionId']): ix for ix, r in enumerate(submJson)} + detQuestions = sorted([r['questionId'] for r in submJson]) + + if ((gtQuestions == detQuestions) is False): + raise Exception('The Det file is not valid. Question IDs must much GT') + + for gtObject in gtJson['data']: + + try: + q_id = int(gtObject['questionId']) + res_ix = res_id_to_index[q_id] + + except: + raise Exception('The Det file is not valid. Question ' + str(gtObject['questionId']) + ' not present') + + else: + detObject = submJson[res_ix] + + # if detObject['questionId'] != gtObject['questionId'] : + # raise Exception("Answer #" + str(i) + " not valid (invalid question ID. Expected:" + str(gtObject['questionId']) + "Found:" + detObject['questionId'] + ")") + + if 'answer' not in detObject: + raise Exception('Question ' + str(gtObject['questionId']) + ' not valid (no answer key)') + + if isinstance(detObject['answer'], list) is True: + raise Exception( + 'Question ' + str(gtObject['questionId']) + ' not valid (answer key has to be a single string)') + + +def evaluate_method(gtFilePath, submFilePath, evaluationParams): + """ + Method evaluate_method: evaluate method and returns the results + Results. Dictionary with the following values: + - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } + - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } + """ + + show_scores_per_answer_type = evaluationParams.answer_types + + gtJson = json.load(open(gtFilePath, 'rb')) + submJson = json.load(open(submFilePath, 'rb')) + + res_id_to_index = {int(r['questionId']): ix for ix, r in enumerate(submJson)} + + perSampleMetrics = {} + + totalScore = 0 + row = 0 + + if show_scores_per_answer_type: + answerTypeTotalScore = {x: 0 for x in answer_types.keys()} + answerTypeNumQuestions = {x: 0 for x in answer_types.keys()} + + evidenceTypeTotalScore = {x: 0 for x in evidence_types.keys()} + evidenceTypeNumQuestions = {x: 0 for x in evidence_types.keys()} + + reasoningTypeTotalScore = {x: 0 for x in reasoning_requirements.keys()} + reasoningTypeNumQuestions = {x: 0 for x in reasoning_requirements.keys()} + + for gtObject in gtJson['data']: + + q_id = int(gtObject['questionId']) + res_ix = res_id_to_index[q_id] + detObject = submJson[res_ix] + + if q_id in question_ids_to_exclude: + question_result = 0 + info = 'Question EXCLUDED from the result' + + else: + info = '' + values = [] + for answer in gtObject['answers']: + # preprocess both the answers - gt and prediction + gt_answer = ' '.join(answer.strip().lower().split()) + det_answer = ' '.join(detObject['answer'].strip().lower().split()) + + # dist = levenshtein_distance(answer.lower(), detObject['answer'].lower()) + dist = levenshtein_distance(gt_answer, det_answer) + length = max(len(answer.upper()), len(detObject['answer'].upper())) + values.append(0.0 if length == 0 else float(dist) / float(length)) + + question_result = 1 - min(values) + + if (question_result < evaluationParams.anls_threshold): + question_result = 0 + + totalScore += question_result + + if show_scores_per_answer_type: + for q_type in gtObject['answer_type']: + answerTypeTotalScore[q_type] += question_result + answerTypeNumQuestions[q_type] += 1 + + for q_type in gtObject['evidence']: + evidenceTypeTotalScore[q_type] += question_result + evidenceTypeNumQuestions[q_type] += 1 + + for q_type in gtObject['operation/reasoning']: + reasoningTypeTotalScore[q_type] += question_result + reasoningTypeNumQuestions[q_type] += 1 + + perSampleMetrics[str(gtObject['questionId'])] = { + 'score': question_result, + 'question': gtObject['question'], + 'gt': gtObject['answers'], + 'det': detObject['answer'], + 'info': info + } + row = row + 1 + + methodMetrics = { + 'score': 0 if len(gtJson['data']) == 0 else totalScore / (len(gtJson['data']) - len(question_ids_to_exclude)) + } + + answer_types_scores = {} + evidence_types_scores = {} + operation_types_scores = {} + + if show_scores_per_answer_type: + for a_type, ref in answer_types.items(): + answer_types_scores[ref] = 0 if len(gtJson['data']) == 0 else answerTypeTotalScore[a_type] / ( + answerTypeNumQuestions[a_type]) + + for e_type, ref in evidence_types.items(): + evidence_types_scores[ref] = 0 if len(gtJson['data']) == 0 else evidenceTypeTotalScore[e_type] / ( + evidenceTypeNumQuestions[e_type]) + + for r_type, ref in reasoning_requirements.items(): + operation_types_scores[ref] = 0 if len(gtJson['data']) == 0 else reasoningTypeTotalScore[r_type] / ( + reasoningTypeNumQuestions[r_type]) + + resDict = { + 'result': methodMetrics, + 'scores_by_types': {'answer_types': answer_types_scores, 'evidence_types': evidence_types_scores, + 'operation_types': operation_types_scores}, + 'per_sample_result': perSampleMetrics + } + + return resDict + + +def display_results(results, show_answer_types): + print('\nOverall ANLS: {:2.4f}'.format(results['result']['score'])) + + if show_answer_types: + print('\nAnswer types:') + for a_type in answer_types.values(): + print('\t{:12s} {:2.4f}'.format(a_type, results['scores_by_types']['answer_types'][a_type])) + + print('\nEvidence types:') + for e_type in evidence_types.values(): + print('\t{:12s} {:2.4f}'.format(e_type, results['scores_by_types']['evidence_types'][e_type])) + + print('\nOperation required:') + for r_type in reasoning_requirements.values(): + print('\t{:12s} {:2.4f}'.format(r_type, results['scores_by_types']['operation_types'][r_type])) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='InfographVQA evaluation script.') + + parser.add_argument('-g', '--ground_truth', type=str, help='Path of the Ground Truth file.', required=True) + parser.add_argument('-s', '--submission_file', type=str, help="Path of your method's results file.", required=True) + + parser.add_argument('-t', '--anls_threshold', type=float, default=0.5, + help='ANLS threshold to use (See Scene-Text VQA paper for more info.).', required=False) + parser.add_argument('-a', '--answer_types', type=bool, default=False, + help='Score break down by answer types (special gt file required).', required=False) + parser.add_argument('-o', '--output', type=str, + help="Path to a directory where to copy the file 'results.json' that contains per-sample results.", + required=False) + + args = parser.parse_args() + + # Validate the format of ground truth and submission files. + validate_data(args.ground_truth, args.submission_file) + + # Evaluate method + results = evaluate_method(args.ground_truth, args.submission_file, args) + + display_results(results, args.answer_types) + + if args.output: + output_dir = args.output + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + resultsOutputname = os.path.join(output_dir, 'results.json') + save_json(resultsOutputname, results) + + print('All results including per-sample result has been correctly saved!') diff --git a/internvl_chat_audio/eval/vqa/textvqa_eval.py b/internvl_chat_audio/eval/vqa/textvqa_eval.py new file mode 100644 index 00000000..3a002f2d --- /dev/null +++ b/internvl_chat_audio/eval/vqa/textvqa_eval.py @@ -0,0 +1,335 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# copied from https://github.com/haotian-liu/LLaVA/blob/main/llava/eval/m4c_evaluator.py +import re + +from tqdm import tqdm + + +class EvalAIAnswerProcessor: + """ + Processes an answer similar to Eval AI + copied from + https://github.com/facebookresearch/mmf/blob/c46b3b3391275b4181567db80943473a89ab98ab/pythia/tasks/processors.py#L897 + """ + + CONTRACTIONS = { + 'aint': "ain't", + 'arent': "aren't", + 'cant': "can't", + 'couldve': "could've", + 'couldnt': "couldn't", + "couldn'tve": "couldn't've", + "couldnt've": "couldn't've", + 'didnt': "didn't", + 'doesnt': "doesn't", + 'dont': "don't", + 'hadnt': "hadn't", + "hadnt've": "hadn't've", + "hadn'tve": "hadn't've", + 'hasnt': "hasn't", + 'havent': "haven't", + 'hed': "he'd", + "hed've": "he'd've", + "he'dve": "he'd've", + 'hes': "he's", + 'howd': "how'd", + 'howll': "how'll", + 'hows': "how's", + "Id've": "I'd've", + "I'dve": "I'd've", + 'Im': "I'm", + 'Ive': "I've", + 'isnt': "isn't", + 'itd': "it'd", + "itd've": "it'd've", + "it'dve": "it'd've", + 'itll': "it'll", + "let's": "let's", + 'maam': "ma'am", + 'mightnt': "mightn't", + "mightnt've": "mightn't've", + "mightn'tve": "mightn't've", + 'mightve': "might've", + 'mustnt': "mustn't", + 'mustve': "must've", + 'neednt': "needn't", + 'notve': "not've", + 'oclock': "o'clock", + 'oughtnt': "oughtn't", + "ow's'at": "'ow's'at", + "'ows'at": "'ow's'at", + "'ow'sat": "'ow's'at", + 'shant': "shan't", + "shed've": "she'd've", + "she'dve": "she'd've", + "she's": "she's", + 'shouldve': "should've", + 'shouldnt': "shouldn't", + "shouldnt've": "shouldn't've", + "shouldn'tve": "shouldn't've", + "somebody'd": 'somebodyd', + "somebodyd've": "somebody'd've", + "somebody'dve": "somebody'd've", + 'somebodyll': "somebody'll", + 'somebodys': "somebody's", + 'someoned': "someone'd", + "someoned've": "someone'd've", + "someone'dve": "someone'd've", + 'someonell': "someone'll", + 'someones': "someone's", + 'somethingd': "something'd", + "somethingd've": "something'd've", + "something'dve": "something'd've", + 'somethingll': "something'll", + 'thats': "that's", + 'thered': "there'd", + "thered've": "there'd've", + "there'dve": "there'd've", + 'therere': "there're", + 'theres': "there's", + 'theyd': "they'd", + "theyd've": "they'd've", + "they'dve": "they'd've", + 'theyll': "they'll", + 'theyre': "they're", + 'theyve': "they've", + 'twas': "'twas", + 'wasnt': "wasn't", + "wed've": "we'd've", + "we'dve": "we'd've", + 'weve': "we've", + 'werent': "weren't", + 'whatll': "what'll", + 'whatre': "what're", + 'whats': "what's", + 'whatve': "what've", + 'whens': "when's", + 'whered': "where'd", + 'wheres': "where's", + 'whereve': "where've", + 'whod': "who'd", + "whod've": "who'd've", + "who'dve": "who'd've", + 'wholl': "who'll", + 'whos': "who's", + 'whove': "who've", + 'whyll': "why'll", + 'whyre': "why're", + 'whys': "why's", + 'wont': "won't", + 'wouldve': "would've", + 'wouldnt': "wouldn't", + "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", + 'yall': "y'all", + "yall'll": "y'all'll", + "y'allll": "y'all'll", + "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", + "y'all'dve": "y'all'd've", + 'youd': "you'd", + "youd've": "you'd've", + "you'dve": "you'd've", + 'youll': "you'll", + 'youre': "you're", + 'youve': "you've", + } + + NUMBER_MAP = { + 'none': '0', + 'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10', + } + ARTICLES = ['a', 'an', 'the'] + PERIOD_STRIP = re.compile(r'(?!<=\d)(\.)(?!\d)') + COMMA_STRIP = re.compile(r'(?<=\d)(\,)+(?=\d)') + PUNCTUATIONS = [ + ';', + r'/', + '[', + ']', + '"', + '{', + '}', + '(', + ')', + '=', + '+', + '\\', + '_', + '-', + '>', + '<', + '@', + '`', + ',', + '?', + '!', + ] + + def __init__(self, *args, **kwargs): + pass + + def word_tokenize(self, word): + word = word.lower() + word = word.replace(',', '').replace('?', '').replace("'s", " 's") + return word.strip() + + def process_punctuation(self, in_text): + out_text = in_text + for p in self.PUNCTUATIONS: + if (p + ' ' in in_text or ' ' + p in in_text) or ( + re.search(self.COMMA_STRIP, in_text) is not None + ): + out_text = out_text.replace(p, '') + else: + out_text = out_text.replace(p, ' ') + out_text = self.PERIOD_STRIP.sub('', out_text, re.UNICODE) + return out_text + + def process_digit_article(self, in_text): + out_text = [] + temp_text = in_text.lower().split() + for word in temp_text: + word = self.NUMBER_MAP.setdefault(word, word) + if word not in self.ARTICLES: + out_text.append(word) + else: + pass + for word_id, word in enumerate(out_text): + if word in self.CONTRACTIONS: + out_text[word_id] = self.CONTRACTIONS[word] + out_text = ' '.join(out_text) + return out_text + + def __call__(self, item): + item = self.word_tokenize(item) + item = item.replace('\n', ' ').replace('\t', ' ').strip() + item = self.process_punctuation(item) + item = self.process_digit_article(item) + return item + + +class TextVQAAccuracyEvaluator: + def __init__(self): + self.answer_processor = EvalAIAnswerProcessor() + + def _compute_answer_scores(self, raw_answers): + """ + compute the accuracy (soft score) of human answers + """ + answers = [self.answer_processor(a) for a in raw_answers] + assert len(answers) == 10 + gt_answers = list(enumerate(answers)) + unique_answers = set(answers) + unique_answer_scores = {} + + for unique_answer in unique_answers: + accs = [] + for gt_answer in gt_answers: + other_answers = [item for item in gt_answers if item != gt_answer] + matching_answers = [ + item for item in other_answers if item[1] == unique_answer + ] + acc = min(1, float(len(matching_answers)) / 3) + accs.append(acc) + unique_answer_scores[unique_answer] = sum(accs) / len(accs) + + return unique_answer_scores + + def eval_pred_list(self, pred_list): + pred_scores = [] + for entry in tqdm(pred_list): + pred_answer = self.answer_processor(entry['pred_answer']) + unique_answer_scores = self._compute_answer_scores(entry['gt_answers']) + score = unique_answer_scores.get(pred_answer, 0.0) + pred_scores.append(score) + + accuracy = sum(pred_scores) / len(pred_scores) + return accuracy + + +class STVQAAccuracyEvaluator: + def __init__(self): + self.answer_processor = EvalAIAnswerProcessor() + + def eval_pred_list(self, pred_list): + pred_scores = [] + for entry in pred_list: + pred_answer = self.answer_processor(entry['pred_answer']) + gts = [self.answer_processor(a) for a in entry['gt_answers']] + score = 1.0 if pred_answer in gts else 0.0 + pred_scores.append(score) + + accuracy = sum(pred_scores) / len(pred_scores) + return accuracy + + +class STVQAANLSEvaluator: + def __init__(self): + import editdistance # install with `pip install editdistance` + + self.get_edit_distance = editdistance.eval + + def get_anls(self, s1, s2): + s1 = s1.lower().strip() + s2 = s2.lower().strip() + iou = 1 - self.get_edit_distance(s1, s2) / max(len(s1), len(s2)) + anls = iou if iou >= 0.5 else 0.0 + return anls + + def eval_pred_list(self, pred_list): + pred_scores = [] + for entry in pred_list: + anls = max( + self.get_anls(entry['pred_answer'], gt) for gt in entry['gt_answers'] + ) + pred_scores.append(anls) + + accuracy = sum(pred_scores) / len(pred_scores) + return accuracy + + +class TextCapsBleu4Evaluator: + def __init__(self): + # The following script requires Java 1.8.0 and pycocotools installed. + # The pycocoevalcap can be installed with pip as + # pip install git+https://github.com/ronghanghu/coco-caption.git@python23 + # Original pycocoevalcap code is at https://github.com/tylin/coco-caption + # but has no python3 support yet. + try: + from pycocoevalcap.bleu.bleu import Bleu + from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer + except ModuleNotFoundError: + print( + 'Please install pycocoevalcap module using ' + 'pip install git+https://github.com/ronghanghu/coco-caption.git@python23' # noqa + ) + raise + + self.tokenizer = PTBTokenizer() + self.scorer = Bleu(4) + + def eval_pred_list(self, pred_list): + # Create reference and hypotheses captions. + gts = {} + res = {} + for idx, entry in enumerate(pred_list): + gts[idx] = [{'caption': a} for a in entry['gt_answers']] + res[idx] = [{'caption': entry['pred_answer']}] + + gts = self.tokenizer.tokenize(gts) + res = self.tokenizer.tokenize(res) + score, _ = self.scorer.compute_score(gts, res) + + bleu4 = score[3] # score is (Bleu-1, Bleu-2, Bleu-3, Bleu-4) + return bleu4 diff --git a/internvl_chat_audio/evaluate.sh b/internvl_chat_audio/evaluate.sh new file mode 100644 index 00000000..dfdb76b4 --- /dev/null +++ b/internvl_chat_audio/evaluate.sh @@ -0,0 +1,583 @@ +set -x + +CHECKPOINT=${1} +DATASET=${2} +CHECKPOINT="$(pwd)/${CHECKPOINT}" +export PYTHONPATH="$(pwd):${PYTHONPATH}" +echo "CHECKPOINT: ${CHECKPOINT}" + +MASTER_PORT=${MASTER_PORT:-63669} +PORT=${PORT:-63665} +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +NODES=$((GPUS / GPUS_PER_NODE)) +export MASTER_PORT=${MASTER_PORT} +export PORT=${PORT} + +# Save original arguments +ARGS=("$@") + +# Parse options +while [[ $# -gt 0 ]]; do + case "$1" in + --auto) + GPUS=1 + shift + ;; + *) + shift + ;; + esac +done +echo "GPUS: ${GPUS}" + +if [ ${DATASET} == "mme" ]; then + cd eval/mme/ + DIRNAME=`basename ${CHECKPOINT}` + python eval.py --checkpoint ${CHECKPOINT} "${ARGS[@]:2}" + python calculation.py --results_dir ${DIRNAME} + cd ../../ +fi + +if [ ${DATASET} == "caption" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/caption/evaluate_caption.py --checkpoint ${CHECKPOINT} "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "caption-coco" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/caption/evaluate_caption.py --checkpoint ${CHECKPOINT} --datasets coco "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "caption-flickr30k" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/caption/evaluate_caption.py --checkpoint ${CHECKPOINT} --datasets flickr30k "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "caption-nocaps" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/caption/evaluate_caption.py --checkpoint ${CHECKPOINT} --datasets nocaps "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-okvqa-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets okvqa_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-textvqa-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets textvqa_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-textvqa-val-ocr" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets textvqa_val_ocr "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-vizwiz-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets vizwiz_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-vizwiz-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets vizwiz_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-vqav2-testdev" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets vqav2_testdev "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-ai2d-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets ai2diagram_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-vqav2-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets vqav2_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-gqa-testdev" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets gqa_testdev_llava "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-docvqa-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets docvqa_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-docvqa-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets docvqa_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-chartqa-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets chartqa_test_human,chartqa_test_augmented "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-infovqa-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets infographicsvqa_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-infovqa-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets infographicsvqa_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-chartqa-test-human" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets chartqa_test_human "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-chartqa-test-augmented" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets chartqa_test_augmented "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-ocrvqa-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets ocrvqa_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "vqa-ocrvqa-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/vqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets ocrvqa_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcoco" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcoco-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcoco_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcoco-testA" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcoco_testA "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcoco-testB" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcoco_testB "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcoco+-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcoco+_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcoco+-testA" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcoco+_testA "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcoco+-testB" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcoco+_testB "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcocog-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcocog_val "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "refcocog-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --datasets refcocog_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "llava-bench" ]; then + rm -rf results/llava_bench_results_review.jsonl + python eval/llava_bench/evaluate_llava_bench.py --checkpoint ${CHECKPOINT} "${ARGS[@]:2}" + python -u eval/llava_bench/eval_gpt_review_bench.py \ + --question data/llava-bench-in-the-wild/questions.jsonl \ + --context data/llava-bench-in-the-wild/context.jsonl \ + --rule eval/llava_bench/rule.json \ + --answer-list \ + data/llava-bench-in-the-wild/answers_gpt4.jsonl \ + results/llava_bench_results.jsonl \ + --output \ + results/llava_bench_results_review.jsonl + python -u eval/llava_bench/summarize_gpt_review.py -f results/llava_bench_results_review.jsonl +fi + +if [ ${DATASET} == "pope" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/pope/evaluate_pope.py --checkpoint ${CHECKPOINT} --datasets pope "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "tiny_lvlm" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/tiny_lvlm/evaluate_lvlm.py --checkpoint ${CHECKPOINT} --datasets updated_datasets "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmvet" ]; then + python eval/mmvet/evaluate_mmvet.py --checkpoint ${CHECKPOINT} --datasets mmvet "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "cmmmu" ]; then + CUDA_VISIBLE_DEVICES=0 python eval/cmmmu/evaluate_cmmmu.py --checkpoint ${CHECKPOINT} --datasets art_and_design "${ARGS[@]:2}" & + CUDA_VISIBLE_DEVICES=1 python eval/cmmmu/evaluate_cmmmu.py --checkpoint ${CHECKPOINT} --datasets business "${ARGS[@]:2}" & + CUDA_VISIBLE_DEVICES=2 python eval/cmmmu/evaluate_cmmmu.py --checkpoint ${CHECKPOINT} --datasets health_and_medicine "${ARGS[@]:2}" & + CUDA_VISIBLE_DEVICES=3 python eval/cmmmu/evaluate_cmmmu.py --checkpoint ${CHECKPOINT} --datasets humanities_and_social_sciences "${ARGS[@]:2}" & + CUDA_VISIBLE_DEVICES=4 python eval/cmmmu/evaluate_cmmmu.py --checkpoint ${CHECKPOINT} --datasets science "${ARGS[@]:2}" & + CUDA_VISIBLE_DEVICES=5 python eval/cmmmu/evaluate_cmmmu.py --checkpoint ${CHECKPOINT} --datasets technology_and_engineering "${ARGS[@]:2}" & + wait +fi + +if [ ${DATASET} == "mmbench-dev-en" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --datasets mmbench_dev_20230712 "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmbench-dev-cn" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --datasets mmbench_dev_cn_20231003 "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmbench-test-en" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --datasets mmbench_test_en_20231003 "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmbench-test-cn" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --datasets mmbench_test_cn_20231003 "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "ccbench-dev" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --datasets ccbench_dev_cn "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "scienceqa" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/scienceqa/evaluate_scienceqa.py --checkpoint ${CHECKPOINT} --datasets sqa_test "${ARGS[@]:2}" +fi + + +if [ ${DATASET} == "mmmu-dev" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmmu/evaluate_mmmu.py --checkpoint ${CHECKPOINT} --datasets MMMU_dev "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmmu-val" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmmu/evaluate_mmmu.py --checkpoint ${CHECKPOINT} --datasets MMMU_validation "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmmu-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmmu/evaluate_mmmu.py --checkpoint ${CHECKPOINT} --datasets MMMU_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmmu-dev-cot" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmmu/evaluate_mmmu_cot.py --checkpoint ${CHECKPOINT} --datasets MMMU_dev "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmmu-val-cot" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmmu/evaluate_mmmu_cot.py --checkpoint ${CHECKPOINT} --datasets MMMU_validation "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mmmu-test-cot" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmmu/evaluate_mmmu_cot.py --checkpoint ${CHECKPOINT} --datasets MMMU_test "${ARGS[@]:2}" +fi + + +if [ ${DATASET} == "mmvp" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mmvp/evaluate_mmvp.py --checkpoint ${CHECKPOINT} --datasets MMVP "${ARGS[@]:2}" +fi + + +if [ ${DATASET} == "mathvista-testmini" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mathvista/evaluate_mathvista.py --checkpoint ${CHECKPOINT} --datasets MathVista_testmini "${ARGS[@]:2}" +fi + + +if [ ${DATASET} == "mathvista-test" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mathvista/evaluate_mathvista.py --checkpoint ${CHECKPOINT} --datasets MathVista_test "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "seed" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/seed/evaluate_seed.py --checkpoint ${CHECKPOINT} --datasets SEEDv1 "${ARGS[@]:2}" +fi + +if [ ${DATASET} == "mvbench" ]; then + torchrun \ + --nnodes=1 \ + --node_rank=0 \ + --master_addr=127.0.0.1 \ + --nproc_per_node=${GPUS} \ + --master_port=${MASTER_PORT} \ + eval/mvbench/evaluate_mvbench.py --checkpoint ${CHECKPOINT} "${ARGS[@]:2}" +fi diff --git a/internvl_chat_audio/examples/image1.jpg b/internvl_chat_audio/examples/image1.jpg new file mode 100644 index 00000000..fd9891ef Binary files /dev/null and b/internvl_chat_audio/examples/image1.jpg differ diff --git a/internvl_chat_audio/examples/image2.jpg b/internvl_chat_audio/examples/image2.jpg new file mode 100644 index 00000000..25948c75 Binary files /dev/null and b/internvl_chat_audio/examples/image2.jpg differ diff --git a/internvl_chat_audio/examples/image3.jpg b/internvl_chat_audio/examples/image3.jpg new file mode 100644 index 00000000..972d5694 Binary files /dev/null and b/internvl_chat_audio/examples/image3.jpg differ diff --git a/internvl_chat_audio/examples/image4.jpg b/internvl_chat_audio/examples/image4.jpg new file mode 100644 index 00000000..0ed2c6c8 Binary files /dev/null and b/internvl_chat_audio/examples/image4.jpg differ diff --git a/internvl_chat_audio/examples/image5.jpg b/internvl_chat_audio/examples/image5.jpg new file mode 100644 index 00000000..bb9c76bf Binary files /dev/null and b/internvl_chat_audio/examples/image5.jpg differ diff --git a/internvl_chat_audio/internvl/conversation.py b/internvl_chat_audio/internvl/conversation.py new file mode 100644 index 00000000..defcfc5c --- /dev/null +++ b/internvl_chat_audio/internvl/conversation.py @@ -0,0 +1,406 @@ +""" +Conversation prompt templates. + +We kindly request that you import fastchat instead of copying this file if you wish to use it. +If you have changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates. +""" + +import dataclasses +from enum import IntEnum, auto +from typing import Any, Dict, List, Tuple, Union + + +class SeparatorStyle(IntEnum): + """Separator styles.""" + + ADD_COLON_SINGLE = auto() + ADD_COLON_TWO = auto() + ADD_COLON_SPACE_SINGLE = auto() + NO_COLON_SINGLE = auto() + NO_COLON_TWO = auto() + ADD_NEW_LINE_SINGLE = auto() + LLAMA2 = auto() + CHATGLM = auto() + CHATML = auto() + CHATINTERN = auto() + DOLLY = auto() + RWKV = auto() + PHOENIX = auto() + ROBIN = auto() + FALCON_CHAT = auto() + CHATGLM3 = auto() + INTERNVL_ZH = auto() + MPT = auto() + + +@dataclasses.dataclass +class Conversation: + """A class that manages prompt templates and keeps all conversation history.""" + + # The name of this template + name: str + # The template of the system prompt + system_template: str = '{system_message}' + # The system message + system_message: str = '' + # The names of two roles + roles: Tuple[str] = ('USER', 'ASSISTANT') + # All messages. Each item is (role, message). + messages: List[List[str]] = () + # The number of few shot examples + offset: int = 0 + # The separator style and configurations + sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE + sep: str = '\n' + sep2: str = None + # Stop criteria (the default one is EOS token) + stop_str: Union[str, List[str]] = None + # Stops generation if meeting any token in this list + stop_token_ids: List[int] = None + + def get_prompt(self) -> str: + """Get the prompt for generation.""" + system_prompt = self.system_template.format(system_message=self.system_message) + if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE: + ret = system_prompt + self.sep + for role, message in self.messages: + if message: + ret += role + ': ' + message + self.sep + else: + ret += role + ':' + return ret + elif self.sep_style == SeparatorStyle.ADD_COLON_TWO: + seps = [self.sep, self.sep2] + ret = system_prompt + seps[0] + for i, (role, message) in enumerate(self.messages): + if message: + ret += role + ': ' + message + seps[i % 2] + else: + ret += role + ':' + return ret + elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE: + ret = system_prompt + self.sep + for role, message in self.messages: + if message: + ret += role + ': ' + message + self.sep + else: + ret += role + ': ' # must be end with a space + return ret + elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE: + ret = '' if system_prompt == '' else system_prompt + self.sep + for role, message in self.messages: + if message: + ret += role + '\n' + message + self.sep + else: + ret += role + '\n' + return ret + elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE: + ret = system_prompt + for role, message in self.messages: + if message: + ret += role + message + self.sep + else: + ret += role + return ret + elif self.sep_style == SeparatorStyle.NO_COLON_TWO: + seps = [self.sep, self.sep2] + ret = system_prompt + for i, (role, message) in enumerate(self.messages): + if message: + ret += role + message + seps[i % 2] + else: + ret += role + return ret + elif self.sep_style == SeparatorStyle.RWKV: + ret = system_prompt + for i, (role, message) in enumerate(self.messages): + if message: + ret += ( + role + + ': ' + + message.replace('\r\n', '\n').replace('\n\n', '\n') + ) + ret += '\n\n' + else: + ret += role + ':' + return ret + elif self.sep_style == SeparatorStyle.LLAMA2: + seps = [self.sep, self.sep2] + if self.system_message: + ret = system_prompt + else: + ret = '[INST] ' + for i, (role, message) in enumerate(self.messages): + tag = self.roles[i % 2] + if message: + if i == 0: + ret += message + ' ' + else: + ret += tag + ' ' + message + seps[i % 2] + else: + ret += tag + return ret + elif self.sep_style == SeparatorStyle.CHATGLM: + # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308 + # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926 + round_add_n = 1 if self.name == 'chatglm2' else 0 + if system_prompt: + ret = system_prompt + self.sep + else: + ret = '' + + for i, (role, message) in enumerate(self.messages): + if i % 2 == 0: + ret += f'[Round {i//2 + round_add_n}]{self.sep}' + + if message: + ret += f'{role}：{message}{self.sep}' + else: + ret += f'{role}：' + return ret + elif self.sep_style == SeparatorStyle.CHATML: + ret = '' if system_prompt == '' else system_prompt + self.sep + '\n' + for role, message in self.messages: + if message: + ret += role + '\n' + message + self.sep + '\n' + else: + ret += role + '\n' + return ret + elif self.sep_style == SeparatorStyle.CHATGLM3: + ret = '' + if self.system_message: + ret += system_prompt + for role, message in self.messages: + if message: + ret += role + '\n' + ' ' + message + else: + ret += role + return ret + elif self.sep_style == SeparatorStyle.CHATINTERN: + # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771 + seps = [self.sep, self.sep2] + ret = system_prompt + for i, (role, message) in enumerate(self.messages): + # if i % 2 == 0: + # ret += "" + if message: + ret += role + ':' + message + seps[i % 2] + '\n' + else: + ret += role + ':' + return ret + elif self.sep_style == SeparatorStyle.DOLLY: + seps = [self.sep, self.sep2] + ret = system_prompt + for i, (role, message) in enumerate(self.messages): + if message: + ret += role + ':\n' + message + seps[i % 2] + if i % 2 == 1: + ret += '\n\n' + else: + ret += role + ':\n' + return ret + elif self.sep_style == SeparatorStyle.PHOENIX: + ret = system_prompt + for role, message in self.messages: + if message: + ret += role + ': ' + '' + message + '' + else: + ret += role + ': ' + '' + return ret + elif self.sep_style == SeparatorStyle.ROBIN: + ret = system_prompt + self.sep + for role, message in self.messages: + if message: + ret += role + ':\n' + message + self.sep + else: + ret += role + ':\n' + return ret + elif self.sep_style == SeparatorStyle.FALCON_CHAT: + ret = '' + if self.system_message: + ret += system_prompt + self.sep + for role, message in self.messages: + if message: + ret += role + ': ' + message + self.sep + else: + ret += role + ':' + + return ret + elif self.sep_style == SeparatorStyle.INTERNVL_ZH: + seps = [self.sep2, self.sep] + ret = self.system_message + seps[0] + for i, (role, message) in enumerate(self.messages): + if message: + ret += role + ': ' + message + seps[i % 2] + else: + ret += role + ':' + return ret + elif self.sep_style == SeparatorStyle.MPT: + ret = system_prompt + self.sep + for role, message in self.messages: + if message: + if type(message) is tuple: + message, _, _ = message + ret += role + message + self.sep + else: + ret += role + return ret + else: + raise ValueError(f'Invalid style: {self.sep_style}') + + def set_system_message(self, system_message: str): + """Set the system message.""" + self.system_message = system_message + + def append_message(self, role: str, message: str): + """Append a new message.""" + self.messages.append([role, message]) + + def update_last_message(self, message: str): + """Update the last output. + + The last message is typically set to be None when constructing the prompt, + so we need to update it in-place after getting the response from a model. + """ + self.messages[-1][1] = message + + def to_gradio_chatbot(self): + """Convert the conversation to gradio chatbot format.""" + ret = [] + for i, (role, msg) in enumerate(self.messages[self.offset :]): + if i % 2 == 0: + ret.append([msg, None]) + else: + ret[-1][-1] = msg + return ret + + def to_openai_api_messages(self): + """Convert the conversation to OpenAI chat completion format.""" + ret = [{'role': 'system', 'content': self.system_message}] + + for i, (_, msg) in enumerate(self.messages[self.offset :]): + if i % 2 == 0: + ret.append({'role': 'user', 'content': msg}) + else: + if msg is not None: + ret.append({'role': 'assistant', 'content': msg}) + return ret + + def copy(self): + return Conversation( + name=self.name, + system_template=self.system_template, + system_message=self.system_message, + roles=self.roles, + messages=[[x, y] for x, y in self.messages], + offset=self.offset, + sep_style=self.sep_style, + sep=self.sep, + sep2=self.sep2, + stop_str=self.stop_str, + stop_token_ids=self.stop_token_ids, + ) + + def dict(self): + return { + 'template_name': self.name, + 'system_message': self.system_message, + 'roles': self.roles, + 'messages': self.messages, + 'offset': self.offset, + } + + +# A global registry for all conversation templates +conv_templates: Dict[str, Conversation] = {} + + +def register_conv_template(template: Conversation, override: bool = False): + """Register a new conversation template.""" + if not override: + assert ( + template.name not in conv_templates + ), f'{template.name} has been registered.' + + conv_templates[template.name] = template + + +def get_conv_template(name: str) -> Conversation: + """Get a conversation template.""" + return conv_templates[name].copy() + + +# InternVL-Chat-V1-1 template +register_conv_template( + Conversation( + name='internvl_zh', + system_template='', + roles=('', ''), + sep_style=SeparatorStyle.INTERNVL_ZH, + sep='', + sep2=' ', + ) +) + + +# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference +# is that during training, the preprocessing function for the Hermes-2 template doesn't add +# at the beginning of the tokenized sequence, while the internlm2-chat template does. +# Therefore, they are completely equivalent during inference. +register_conv_template( + Conversation( + name='Hermes-2', + system_template='<|im_start|>system\n{system_message}', + # note: The new system prompt was not used here to avoid changes in benchmark performance. + # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。', + system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。', + roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), + sep_style=SeparatorStyle.MPT, + sep='<|im_end|>', + stop_token_ids=[ + 2, + 6, + 7, + 8, + ], + stop_str='<|endoftext|>', + ) +) + + +register_conv_template( + Conversation( + name='internlm2-chat', + system_template='<|im_start|>system\n{system_message}', + # note: The new system prompt was not used here to avoid changes in benchmark performance. + # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。', + system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。', + roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), + sep_style=SeparatorStyle.MPT, + sep='<|im_end|>', + stop_token_ids=[ + 2, + 92543, + 92542 + ] + ) +) + + +register_conv_template( + Conversation( + name='phi3-chat', + system_template='<|system|>\n{system_message}', + # note: The new system prompt was not used here to avoid changes in benchmark performance. + # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。', + system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。', + roles=('<|user|>\n', '<|assistant|>\n'), + sep_style=SeparatorStyle.MPT, + sep='<|end|>', + stop_token_ids=[ + 2, + 32000, + 32007 + ] + ) +) diff --git a/internvl_chat_audio/internvl/dist_utils.py b/internvl_chat_audio/internvl/dist_utils.py new file mode 100644 index 00000000..0eb8ae27 --- /dev/null +++ b/internvl_chat_audio/internvl/dist_utils.py @@ -0,0 +1,104 @@ +import os +import socket +import subprocess +from datetime import timedelta + +import deepspeed +import torch +import torch.multiprocessing as mp +from torch import distributed as dist + +timeout = timedelta(minutes=60) + + +def _find_free_port(): + # Copied from https://github.com/facebookresearch/detectron2/blob/main/detectron2/engine/launch.py # noqa: E501 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Binding to port 0 will cause the OS to find an available port for us + sock.bind(('', 0)) + port = sock.getsockname()[1] + sock.close() + # NOTE: there is still a chance the port could be taken by other processes. + return port + + +def _is_free_port(port): + ips = socket.gethostbyname_ex(socket.gethostname())[-1] + ips.append('localhost') + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return all(s.connect_ex((ip, port)) != 0 for ip in ips) + + +def init_dist(launcher, backend='nccl', **kwargs): + if mp.get_start_method(allow_none=True) is None: + mp.set_start_method('spawn') + if launcher == 'pytorch': + _init_dist_pytorch(backend, **kwargs) + elif launcher == 'mpi': + _init_dist_mpi(backend, **kwargs) + elif launcher == 'slurm': + _init_dist_slurm(backend, **kwargs) + else: + raise ValueError(f'Invalid launcher type: {launcher}') + + +def _init_dist_pytorch(backend, **kwargs): + # TODO: use local_rank instead of rank % num_gpus + rank = int(os.environ['RANK']) + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(rank % num_gpus) + # dist.init_process_group(backend=backend, **kwargs) + deepspeed.init_distributed(dist_backend=backend) + + +def _init_dist_mpi(backend, **kwargs): + local_rank = int(os.environ['OMPI_COMM_WORLD_LOCAL_RANK']) + torch.cuda.set_device(local_rank) + if 'MASTER_PORT' not in os.environ: + # 29500 is torch.distributed default port + os.environ['MASTER_PORT'] = '29500' + if 'MASTER_ADDR' not in os.environ: + raise KeyError('The environment variable MASTER_ADDR is not set') + os.environ['WORLD_SIZE'] = os.environ['OMPI_COMM_WORLD_SIZE'] + os.environ['RANK'] = os.environ['OMPI_COMM_WORLD_RANK'] + dist.init_process_group(backend=backend, **kwargs) + + +def _init_dist_slurm(backend, port=None): + """Initialize slurm distributed training environment. + + If argument ``port`` is not specified, then the master port will be system + environment variable ``MASTER_PORT``. If ``MASTER_PORT`` is not in system + environment variable, then a default port ``29500`` will be used. + + Args: + backend (str): Backend of torch.distributed. + port (int, optional): Master port. Defaults to None. + """ + proc_id = int(os.environ['SLURM_PROCID']) + ntasks = int(os.environ['SLURM_NTASKS']) + node_list = os.environ['SLURM_NODELIST'] + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(proc_id % num_gpus) + addr = subprocess.getoutput( + f'scontrol show hostname {node_list} | head -n1') + # specify master port + if port is not None: + os.environ['MASTER_PORT'] = str(port) + elif 'MASTER_PORT' in os.environ: + pass # use MASTER_PORT in the environment variable + else: + # if torch.distributed default port(29500) is available + # then use it, else find a free port + if _is_free_port(29500): + os.environ['MASTER_PORT'] = '29500' + else: + os.environ['MASTER_PORT'] = str(_find_free_port()) + # use MASTER_ADDR in the environment variable if it already exists + if 'MASTER_ADDR' not in os.environ: + os.environ['MASTER_ADDR'] = addr + os.environ['WORLD_SIZE'] = str(ntasks) + os.environ['LOCAL_RANK'] = str(proc_id % num_gpus) + os.environ['RANK'] = str(proc_id) + # dist.init_process_group(backend=backend, timeout=timeout) + deepspeed.init_distributed(dist_backend=backend) diff --git a/internvl_chat_audio/internvl/model/__init__.py b/internvl_chat_audio/internvl/model/__init__.py new file mode 100644 index 00000000..b6beb3ef --- /dev/null +++ b/internvl_chat_audio/internvl/model/__init__.py @@ -0,0 +1,44 @@ +import math + +import torch +from internvl.model.internvl_chat import InternVLChatConfig, InternVLChatModel +from transformers import AutoTokenizer + + +def split_model(num_layers, vit_alpha=0.5): + device_map = {} + world_size = torch.cuda.device_count() + # Since the first GPU will be used for ViT, treat it as half a GPU. + num_layers_per_gpu = math.ceil(num_layers / (world_size - vit_alpha)) + num_layers_per_gpu = [num_layers_per_gpu] * world_size + num_layers_per_gpu[0] = math.ceil(num_layers_per_gpu[0] * (1 - vit_alpha)) + layer_cnt = 0 + for i, num_layer in enumerate(num_layers_per_gpu): + for j in range(num_layer): + device_map[f'language_model.model.layers.{layer_cnt}'] = i + layer_cnt += 1 + device_map['vision_model'] = 0 + device_map['mlp1'] = 0 + device_map['language_model.model.tok_embeddings'] = 0 + device_map['language_model.model.embed_tokens'] = 0 + device_map['language_model.output'] = 0 + device_map['language_model.model.norm'] = 0 + device_map['language_model.lm_head'] = 0 + device_map[f'language_model.model.layers.{num_layers - 1}'] = 0 + + return device_map + + +def load_model_and_tokenizer(args): + if args.auto: + config = InternVLChatConfig.from_pretrained(args.checkpoint) + num_hidden_layers = config.llm_config.num_hidden_layers + device_map = split_model(num_hidden_layers) + kwargs = {'device_map': device_map} if args.auto else {} + tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False) + model = InternVLChatModel.from_pretrained( + args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16, + load_in_8bit=args.load_in_8bit, load_in_4bit=args.load_in_4bit, **kwargs).eval() + if not args.load_in_8bit and not args.load_in_4bit and not args.auto: + model = model.cuda() + return model, tokenizer diff --git a/internvl_chat_audio/internvl/model/audio/configuration_whisper.py b/internvl_chat_audio/internvl/model/audio/configuration_whisper.py new file mode 100644 index 00000000..e221450c --- /dev/null +++ b/internvl_chat_audio/internvl/model/audio/configuration_whisper.py @@ -0,0 +1,340 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Whisper model configuration""" + +from collections import OrderedDict +from typing import TYPE_CHECKING, Any, Mapping, Optional, Union + +from transformers.configuration_utils import PretrainedConfig +from transformers.onnx import OnnxConfig, OnnxSeq2SeqConfigWithPast +from transformers.utils import logging + + +if TYPE_CHECKING: + from transformers.feature_extraction_utils import FeatureExtractionMixin + from transformers.tokenization_utils_base import PreTrainedTokenizerBase + from transformers.utils import TensorType + +logger = logging.get_logger(__name__) + + +# fmt: off +NON_SPEECH_TOKENS = [ + 1, 2, 7, 8, 9, 10, 14, 25, + 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, + 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, + 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, + 1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211, + 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, + 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, + 17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, + 34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361 +] +NON_SPEECH_TOKENS_MULTI = [ + 1, 2, 7, 8, 9, 10, 14, 25, + 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, + 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, + 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, + 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, + 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, + 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, + 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, + 42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362 +] +# fmt: on + + +class WhisperConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`WhisperModel`]. It is used to instantiate a + Whisper model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the Whisper + [openai/whisper-tiny](https://huggingface.co/openai/whisper-tiny) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 51865): + Vocabulary size of the Whisper model. Defines the number of different tokens that can be represented by the + `decoder_input_ids` passed when calling [`WhisperModel`] + num_mel_bins (`int`, *optional*, defaults to 80): + Number of mel features used per input features. Should correspond to the value used in the + `WhisperProcessor` class. + encoder_layers (`int`, *optional*, defaults to 4): + Number of encoder layers. + decoder_layers (`int`, *optional*, defaults to 4): + Number of decoder layers. + encoder_attention_heads (`int`, *optional*, defaults to 6): + Number of attention heads for each attention layer in the Transformer encoder. + decoder_attention_heads (`int`, *optional*, defaults to 6): + Number of attention heads for each attention layer in the Transformer decoder. + encoder_ffn_dim (`int`, *optional*, defaults to 1536): + Dimensionality of the "intermediate" (often named feed-forward) layer in encoder. + decoder_ffn_dim (`int`, *optional*, defaults to 1536): + Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. + encoder_layerdrop (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + decoder_layerdrop (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + decoder_start_token_id (`int`, *optional*, defaults to 50257): + Corresponds to the "<|startoftranscript|>" token, which is automatically used when no `decoder_input_ids` + are provided to the `generate` function. It is used to guide the model`s generation process depending on + the task. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + is_encoder_decoder (`bool`, *optional*, defaults to `True`): + Whether the model is used as an encoder/decoder or not. + activation_function (`str`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + d_model (`int`, *optional*, defaults to 384): + Dimensionality of the layers. + dropout (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + activation_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + init_std (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + scale_embedding (`bool`, *optional*, defaults to False): + Scale embeddings by diving by sqrt(d_model). + max_source_positions (`int`, *optional*, defaults to 1500): + The maximum sequence length of log-mel filter-bank features that this model might ever be used with. + max_target_positions (`int`, *optional*, defaults to 448): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + pad_token_id (`int`, *optional*, defaults to 50256): + Padding token id. + bos_token_id (`int`, *optional*, defaults to 50256): + Begin of stream token id. + eos_token_id (`int`, *optional*, defaults to 50256): + End of stream token id. + suppress_tokens (`List[int]`, *optional*): + A list containing the non-speech tokens that will be used by the logit processor in the `generate` + function. NON_SPEECH_TOKENS and NON_SPEECH_TOKENS_MULTI each correspond to the `english-only` and the + `multilingual` model. + begin_suppress_tokens (`List[int]`, *optional*, defaults to `[220,50256]`): + A list containing tokens that will be supressed at the beginning of the sampling process. Initialized as + the token for `" "` (`blank_token_id`) and the `eos_token_id` + use_weighted_layer_sum (`bool`, *optional*, defaults to `False`): + Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an + instance of [`WhisperForAudioClassification`]. + classifier_proj_size (`int`, *optional*, defaults to 256): + Dimensionality of the projection before token mean-pooling for classification. Only relevant when using an + instance of [`WhisperForAudioClassification`]. + apply_spec_augment (`bool`, *optional*, defaults to `False`): + Whether to apply *SpecAugment* data augmentation to the outputs of the feature encoder. For reference see + [SpecAugment: A Simple Data Augmentation Method for Automatic Speech + Recognition](https://arxiv.org/abs/1904.08779). + mask_time_prob (`float`, *optional*, defaults to 0.05): + Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking + procecure generates `mask_time_prob*len(time_axis)/mask_time_length` independent masks over the axis. If + reasoning from the propability of each feature vector to be chosen as the start of the vector span to be + masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the + actual percentage of masked vectors. This is only relevant if `apply_spec_augment == True`. + mask_time_length (`int`, *optional*, defaults to 10): + Length of vector span along the time axis. + mask_time_min_masks (`int`, *optional*, defaults to 2),: + The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step, + irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length < + mask_time_min_masks'' + mask_feature_prob (`float`, *optional*, defaults to 0.0): + Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The + masking procecure generates `mask_feature_prob*len(feature_axis)/mask_time_length` independent masks over + the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector + span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap + may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is + True`. + mask_feature_length (`int`, *optional*, defaults to 10): + Length of vector span along the feature axis. + mask_feature_min_masks (`int`, *optional*, defaults to 0),: + The minimum number of masks of length `mask_feature_length` generated along the feature axis, each time + step, irrespectively of `mask_feature_prob`. Only relevant if + `mask_feature_prob*len(feature_axis)/mask_feature_length < mask_feature_min_masks`. + median_filter_width (`int`, *optional*, defaults to 7): + Width of the median filter used to smoothen to cross-attention outputs when computing token timestamps. + Should be an odd number. + + Example: + + ```python + >>> from transformers import WhisperConfig, WhisperModel + + >>> # Initializing a Whisper tiny style configuration + >>> configuration = WhisperConfig() + + >>> # Initializing a model (with random weights) from the tiny style configuration + >>> model = WhisperModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "whisper" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"} + + def __init__( + self, + vocab_size=51865, + num_mel_bins=80, + encoder_layers=4, + encoder_attention_heads=6, + decoder_layers=4, + decoder_attention_heads=6, + decoder_ffn_dim=1536, + encoder_ffn_dim=1536, + encoder_layerdrop=0.0, + decoder_layerdrop=0.0, + decoder_start_token_id=50257, + use_cache=True, + is_encoder_decoder=True, + activation_function="gelu", + d_model=384, + dropout=0.0, + attention_dropout=0.0, + activation_dropout=0.0, + init_std=0.02, + scale_embedding=False, + max_source_positions=1500, + max_target_positions=448, + pad_token_id=50256, + bos_token_id=50256, + eos_token_id=50256, + suppress_tokens=None, + begin_suppress_tokens=[220, 50256], + use_weighted_layer_sum=False, + classifier_proj_size=256, + apply_spec_augment=False, + mask_time_prob=0.05, + mask_time_length=10, + mask_time_min_masks=2, + mask_feature_prob=0.0, + mask_feature_length=10, + mask_feature_min_masks=0, + median_filter_width=7, + **kwargs, + ): + self.vocab_size = vocab_size + self.num_mel_bins = num_mel_bins + self.d_model = d_model + self.encoder_layers = encoder_layers + self.encoder_attention_heads = encoder_attention_heads + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.decoder_ffn_dim = decoder_ffn_dim + self.encoder_ffn_dim = encoder_ffn_dim + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation_dropout = activation_dropout + self.activation_function = activation_function + self.init_std = init_std + self.encoder_layerdrop = encoder_layerdrop + self.decoder_layerdrop = decoder_layerdrop + self.use_cache = use_cache + self.num_hidden_layers = encoder_layers + self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + self.max_source_positions = max_source_positions + self.max_target_positions = max_target_positions + + # Audio Classification-specific parameters. Feel free to ignore for other classes. + self.classifier_proj_size = classifier_proj_size + self.use_weighted_layer_sum = use_weighted_layer_sum + + # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779 + self.apply_spec_augment = apply_spec_augment + self.mask_time_prob = mask_time_prob + self.mask_time_length = mask_time_length + self.mask_time_min_masks = mask_time_min_masks + self.mask_feature_prob = mask_feature_prob + self.mask_feature_length = mask_feature_length + self.mask_feature_min_masks = mask_feature_min_masks + + self.median_filter_width = median_filter_width + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + is_encoder_decoder=is_encoder_decoder, + decoder_start_token_id=decoder_start_token_id, + suppress_tokens=suppress_tokens, + begin_suppress_tokens=begin_suppress_tokens, + **kwargs, + ) + + +class WhisperOnnxConfig(OnnxSeq2SeqConfigWithPast): + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + common_inputs = OrderedDict( + [ + ("input_features", {0: "batch", 1: "feature_size", 2: "encoder_sequence"}), + ] + ) + if self.use_past: + common_inputs["decoder_input_ids"] = {0: "batch"} + else: + common_inputs["decoder_input_ids"] = {0: "batch", 1: "decoder_sequence"} + + if self.use_past: + self.fill_with_past_key_values_(common_inputs, direction="inputs") + + return common_inputs + + def generate_dummy_inputs( + self, + preprocessor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"], + batch_size: int = -1, + seq_length: int = -1, + is_pair: bool = False, + framework: Optional["TensorType"] = None, + sampling_rate: int = 22050, + time_duration: float = 5.0, + frequency: int = 220, + ) -> Mapping[str, Any]: + dummy_inputs = OrderedDict() + encoder_inputs = OnnxConfig.generate_dummy_inputs( + self, + preprocessor=preprocessor.feature_extractor, + batch_size=batch_size, + framework=framework, + sampling_rate=sampling_rate, + time_duration=time_duration, + frequency=frequency, + ) + encoder_sequence_length = encoder_inputs["input_features"].shape[2] + seq_length = encoder_sequence_length // 2 if self.use_past else seq_length + + decoder_inputs = super().generate_dummy_inputs( + preprocessor.tokenizer, batch_size, seq_length, is_pair, framework + ) + + dummy_inputs["input_features"] = encoder_inputs.pop("input_features") + dummy_inputs["decoder_input_ids"] = decoder_inputs.pop("decoder_input_ids") + + if "past_key_values" in decoder_inputs: + dummy_inputs["past_key_values"] = decoder_inputs.pop("past_key_values") + + return dummy_inputs + + @property + def atol_for_validation(self) -> float: + return 1e-3 \ No newline at end of file diff --git a/internvl_chat_audio/internvl/model/audio/modeling_whisper.py b/internvl_chat_audio/internvl/model/audio/modeling_whisper.py new file mode 100644 index 00000000..492451d8 --- /dev/null +++ b/internvl_chat_audio/internvl/model/audio/modeling_whisper.py @@ -0,0 +1,2304 @@ +# coding=utf-8 +# Copyright 2022 The OpenAI Authors and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch Whisper model.""" +import math +from typing import Optional, Tuple, Union +from transformers.utils import ModelOutput +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss + +from transformers.activations import ACT2FN +from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa +from transformers.modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPastAndCrossAttentions, + CausalLMOutputWithCrossAttentions, + Seq2SeqLMOutput, + Seq2SeqModelOutput, + SequenceClassifierOutput, +) +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import ( + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_flash_attn_2_available, + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, +) +from .configuration_whisper import WhisperConfig +from transformers.models.whisper.generation_whisper import WhisperGenerationMixin + + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + +logger = logging.get_logger(__name__) + +_HIDDEN_STATES_START_POSITION = 1 + +_CONFIG_FOR_DOC = "WhisperConfig" +_CHECKPOINT_FOR_DOC = "openai/whisper-tiny" + + +# Copied from transformers.models.llama.modeling_llama._get_unpad_data +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + + +def sinusoids(length: int, channels: int, max_timescale: float = 10000) -> torch.Tensor: + """Returns sinusoids for positional embedding""" + if channels % 2 != 0: + raise ValueError( + f"Number of channels has to be divisible by 2 for sinusoidal positional embeddings, got {channels} channels." + ) + log_timescale_increment = math.log(max_timescale) / (channels // 2 - 1) + inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2)) + scaled_time = torch.arange(length).view(-1, 1) * inv_timescales.view(1, -1) + return torch.cat([scaled_time.sin(), scaled_time.cos()], dim=1) + + +# Copied from transformers.models.bart.modeling_bart.shift_tokens_right +def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): + """ + Shift input ids one token to the right. + """ + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[:, 1:] = input_ids[:, :-1].clone() + shifted_input_ids[:, 0] = decoder_start_token_id + + if pad_token_id is None: + raise ValueError("self.model.config.pad_token_id has to be defined.") + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + return shifted_input_ids + + +# Copied from transformers.models.wav2vec2.modeling_wav2vec2._compute_mask_indices +def _compute_mask_indices( + shape: Tuple[int, int], + mask_prob: float, + mask_length: int, + attention_mask: Optional[torch.LongTensor] = None, + min_masks: int = 0, +) -> np.ndarray: + """ + Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for + ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on + CPU as part of the preprocessing during training. + + Args: + shape: The shape for which to compute masks. This should be of a tuple of size 2 where + the first element is the batch size and the second element is the length of the axis to span. + mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of + independently generated mask spans of length `mask_length` is computed by + `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the + actual percentage will be smaller. + mask_length: size of the mask + min_masks: minimum number of masked spans + attention_mask: A (right-padded) attention mask which independently shortens the feature axis of + each batch dimension. + """ + batch_size, sequence_length = shape + + if mask_length < 1: + raise ValueError("`mask_length` has to be bigger than 0.") + + if mask_length > sequence_length: + raise ValueError( + f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length}" + f" and `sequence_length`: {sequence_length}`" + ) + + # epsilon is used for probabilistic rounding + epsilon = np.random.rand(1).item() + + def compute_num_masked_span(input_length): + """Given input length, compute how many spans should be masked""" + num_masked_span = int(mask_prob * input_length / mask_length + epsilon) + num_masked_span = max(num_masked_span, min_masks) + + # make sure num masked span <= sequence_length + if num_masked_span * mask_length > sequence_length: + num_masked_span = sequence_length // mask_length + + # make sure num_masked span is also <= input_length - (mask_length - 1) + if input_length - (mask_length - 1) < num_masked_span: + num_masked_span = max(input_length - (mask_length - 1), 0) + + return num_masked_span + + # compute number of masked spans in batch + input_lengths = ( + attention_mask.sum(-1).detach().tolist() + if attention_mask is not None + else [sequence_length for _ in range(batch_size)] + ) + + # SpecAugment mask to fill + spec_aug_mask = np.zeros((batch_size, sequence_length), dtype=bool) + spec_aug_mask_idxs = [] + + max_num_masked_span = compute_num_masked_span(sequence_length) + + if max_num_masked_span == 0: + return spec_aug_mask + + for input_length in input_lengths: + # compute num of masked spans for this input + num_masked_span = compute_num_masked_span(input_length) + + # get random indices to mask + spec_aug_mask_idx = np.random.choice( + np.arange(input_length - (mask_length - 1)), num_masked_span, replace=False + ) + + # pick first sampled index that will serve as a dummy index to pad vector + # to ensure same dimension for all batches due to probabilistic rounding + # Picking first sample just pads those vectors twice. + if len(spec_aug_mask_idx) == 0: + # this case can only happen if `input_length` is strictly smaller then + # `sequence_length` in which case the last token has to be a padding + # token which we can use as a dummy mask id + dummy_mask_idx = sequence_length - 1 + else: + dummy_mask_idx = spec_aug_mask_idx[0] + + spec_aug_mask_idx = np.concatenate( + [spec_aug_mask_idx, np.ones(max_num_masked_span - num_masked_span, dtype=np.int32) * dummy_mask_idx] + ) + spec_aug_mask_idxs.append(spec_aug_mask_idx) + + spec_aug_mask_idxs = np.array(spec_aug_mask_idxs) + + # expand masked indices to masked spans + spec_aug_mask_idxs = np.broadcast_to( + spec_aug_mask_idxs[:, :, None], (batch_size, max_num_masked_span, mask_length) + ) + spec_aug_mask_idxs = spec_aug_mask_idxs.reshape(batch_size, max_num_masked_span * mask_length) + + # add offset to the starting indexes so that indexes now create a span + offsets = np.arange(mask_length)[None, None, :] + offsets = np.broadcast_to(offsets, (batch_size, max_num_masked_span, mask_length)).reshape( + batch_size, max_num_masked_span * mask_length + ) + spec_aug_mask_idxs = spec_aug_mask_idxs + offsets + + # ensure that we cannot have indices larger than sequence_length + if spec_aug_mask_idxs.max() > sequence_length - 1: + spec_aug_mask_idxs[spec_aug_mask_idxs > sequence_length - 1] = sequence_length - 1 + + # scatter indices to mask + np.put_along_axis(spec_aug_mask, spec_aug_mask_idxs, 1, -1) + + return spec_aug_mask + + +class WhisperPositionalEmbedding(nn.Embedding): + def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None): + super().__init__(num_positions, embedding_dim) + + def forward(self, input_ids, past_key_values_length=0, position_ids=None): + if position_ids is None: + return self.weight[past_key_values_length : past_key_values_length + input_ids.shape[1]] + else: + return self.weight[position_ids] + + +class WhisperAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + is_decoder: bool = False, + bias: bool = True, + is_causal: bool = False, + config: Optional[WhisperConfig] = None, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + self.config = config + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + self.is_causal = is_causal + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=False) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + + # Copied from transformers.models.bart.modeling_bart.BartAttention._shape with BART->whisper + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + # Copied from transformers.models.bart.modeling_bart.BartAttention.forward with BART->whisper + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + # `past_key_value[0].shape[2] == key_value_states.shape[1]` + # is checking that the `sequence_length` of the `past_key_value` is the same as + # the provided `key_value_states` to support prefix tuning + if ( + is_cross_attention + and past_key_value is not None + and past_key_value[0].shape[2] == key_value_states.shape[1] + ): + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.reshape(*proj_shape) + value_states = value_states.reshape(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + # if attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + # ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" + f" {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned across GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value + + +# Copied from transformers.models.bart.modeling_bart.BartFlashAttention2 with Bart->Whisper +class WhisperFlashAttention2(WhisperAttention): + """ + Whisper flash attention module. This module inherits from `WhisperAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def _reshape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim) + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # WhisperFlashAttention2 attention does not support output_attentions + if output_attentions: + raise ValueError("WhisperFlashAttention2 attention does not support output_attentions") + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, q_len, _ = hidden_states.size() + + # get query proj + query_states = self._reshape(self.q_proj(hidden_states), -1, bsz) + # get key, value proj + # `past_key_value[0].shape[2] == key_value_states.shape[1]` + # is checking that the `sequence_length` of the `past_key_value` is the same as + # the provided `key_value_states` to support prefix tuning + if ( + is_cross_attention + and past_key_value is not None + and past_key_value[0].shape[2] == key_value_states.shape[1] + ): + # reuse k,v, cross_attentions + key_states = past_key_value[0].transpose(1, 2) + value_states = past_key_value[1].transpose(1, 2) + elif is_cross_attention: + # cross_attentions + key_states = self._reshape(self.k_proj(key_value_states), -1, bsz) + value_states = self._reshape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._reshape(self.k_proj(hidden_states), -1, bsz) + value_states = self._reshape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0].transpose(1, 2), key_states], dim=1) + value_states = torch.cat([past_key_value[1].transpose(1, 2), value_states], dim=1) + else: + # self_attention + key_states = self._reshape(self.k_proj(hidden_states), -1, bsz) + value_states = self._reshape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states.transpose(1, 2), value_states.transpose(1, 2)) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (LlamaRMSNorm handles it correctly) + + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + attn_output = self._flash_attention_forward( + query_states, key_states, value_states, attention_mask, q_len, dropout=self.dropout + ) + + attn_output = attn_output.reshape(bsz, q_len, -1) + attn_output = self.out_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._flash_attention_forward + def _flash_attention_forward( + self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + attn_output = flash_attn_func( + query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal + ) + + return attn_output + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._upad_input + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape + + key_layer = index_first_axis( + key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + value_layer = index_first_axis( + value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + + +class WhisperSdpaAttention(WhisperAttention): + # Copied from transformers.models.bart.modeling_bart.BartSdpaAttention.forward with BART->whisper, Bart->Whisper + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + if output_attentions or layer_head_mask is not None: + # TODO: Improve this warning with e.g. `model.config._attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "WhisperModel is using WhisperSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True` or `layer_head_mask` not None. Falling back to the manual attention" + ' implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states, + key_value_states=key_value_states, + past_key_value=past_key_value, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) + # get key, value proj + # `past_key_value[0].shape[2] == key_value_states.shape[1]` + # is checking that the `sequence_length` of the `past_key_value` is the same as + # the provided `key_value_states` to support prefix tuning + if ( + is_cross_attention + and past_key_value is not None + and past_key_value[0].shape[2] == key_value_states.shape[1] + ): + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + query_states = self._shape(query_states, tgt_len, bsz) + + # NOTE: SDPA with memory-efficient backend is currently (torch==2.1.2) bugged when using non-contiguous inputs and a custom attn_mask, + # but we are fine here as `_shape` do call `.contiguous()`. Reference: https://github.com/pytorch/pytorch/issues/112577 + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=self.dropout if self.training else 0.0, + # The tgt_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case tgt_len == 1. + is_causal=self.is_causal and attention_mask is None and tgt_len > 1, + ) + + if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned across GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, None, past_key_value + + +WHISPER_ATTENTION_CLASSES = { + "eager": WhisperAttention, + "flash_attention_2": WhisperFlashAttention2, + "sdpa": WhisperSdpaAttention, +} + + +# Copied from transformers.models.mbart.modeling_mbart.MBartEncoderLayer with MBart->Whisper, MBART->WHISPER +class WhisperEncoderLayer(nn.Module): + def __init__(self, config: WhisperConfig): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = WHISPER_ATTENTION_CLASSES[config._attn_implementation]( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + dropout=config.attention_dropout, + config=config, + ) + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + layer_head_mask: torch.Tensor, + output_attentions: bool = False, + ) -> torch.Tensor: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size + `(encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states, attn_weights, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + if hidden_states.dtype == torch.float16 and ( + torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any() + ): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +# Copied from transformers.models.mbart.modeling_mbart.MBartDecoderLayer with MBart->Whisper, MBART->WHISPER +class WhisperDecoderLayer(nn.Module): + def __init__(self, config: WhisperConfig): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = WHISPER_ATTENTION_CLASSES[config._attn_implementation]( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + is_decoder=True, + is_causal=True, + config=config, + ) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.encoder_attn = WHISPER_ATTENTION_CLASSES[config._attn_implementation]( + self.embed_dim, + config.decoder_attention_heads, + dropout=config.attention_dropout, + is_decoder=True, + config=config, + ) + self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) + self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + cross_attn_layer_head_mask: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = True, + ) -> torch.Tensor: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + encoder_hidden_states (`torch.FloatTensor`): + cross attention input to the layer of shape `(batch, seq_len, embed_dim)` + encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size + `(encoder_attention_heads,)`. + cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of + size `(decoder_attention_heads,)`. + past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Self Attention + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + # add present self-attn cache to positions 1,2 of present_key_value tuple + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + past_key_value=self_attn_past_key_value, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + # Cross-Attention Block + cross_attn_present_key_value = None + cross_attn_weights = None + if encoder_hidden_states is not None: + residual = hidden_states + hidden_states = self.encoder_attn_layer_norm(hidden_states) + + # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple + cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None + hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn( + hidden_states=hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + layer_head_mask=cross_attn_layer_head_mask, + past_key_value=cross_attn_past_key_value, + output_attentions=output_attentions, + ) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + # add cross-attn to positions 3,4 of present_key_value tuple + present_key_value = present_key_value + cross_attn_present_key_value + + # Fully Connected + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights, cross_attn_weights) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +class WhisperPreTrainedModel(PreTrainedModel): + config_class = WhisperConfig + base_model_prefix = "model" + main_input_name = "input_features" + supports_gradient_checkpointing = True + _no_split_modules = ["WhisperEncoderLayer", "WhisperDecoderLayer"] + _supports_flash_attn_2 = True + _supports_sdpa = True + + def _init_weights(self, module): + std = self.config.init_std + if isinstance(module, (nn.Linear, nn.Conv1d)): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, WhisperEncoder): + with torch.no_grad(): + embed_positions = module.embed_positions.weight + embed_positions.copy_(sinusoids(*embed_positions.shape)) + + def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor): + """ + Computes the output length of the convolutional layers + """ + input_lengths = (input_lengths - 1) // 2 + 1 + + return input_lengths + + +WHISPER_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`WhisperConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +WHISPER_INPUTS_DOCSTRING = r""" + Args: + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing *SpecAugment* data augmentation on padding token indices. Mask values selected in + `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Whisper uses the `decoder_start_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_whisper._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the BART + paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more control over how to convert + `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +WHISPER_ENCODER_INPUTS_DOCSTRING = r""" + Args: + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +class WhisperEncoder(WhisperPreTrainedModel): + """ + Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a + [`WhisperEncoderLayer`]. + + Args: + config: WhisperConfig + """ + + def __init__(self, config: WhisperConfig, avg_pool: bool = True): + super().__init__(config) + self.dropout = config.dropout + self.layerdrop = config.encoder_layerdrop + + embed_dim = config.d_model + self.num_mel_bins = config.num_mel_bins + self.padding_idx = config.pad_token_id + self.max_source_positions = config.max_source_positions + self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0 + + self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1) + self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1) + + self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim) + self.embed_positions.requires_grad_(False) + + self.layers = nn.ModuleList([WhisperEncoderLayer(config) for _ in range(config.encoder_layers)]) + self.layer_norm = nn.LayerNorm(config.d_model) + + self.gradient_checkpointing = False + self.config = config + # Initialize weights and apply final processing + self.post_init() + if avg_pool: + self.avg_pooler = nn.AvgPool1d(2, stride=2) + else: + self.avg_pooler = None + + def _freeze_parameters(self): + for param in self.parameters(): + param.requires_grad = False + self._requires_grad = False + + def get_input_embeddings(self) -> nn.Module: + return self.conv1 + + def set_input_embeddings(self, value: nn.Module): + self.conv1 = value + + def forward( + self, + input_features, + attention_mask=None, + audio_len_after_cnn=None, + head_mask=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + attention_mask (`torch.Tensor`)`, *optional*): + Whisper does not support masking of the `input_features`, this argument is preserved for compatibility, + but it is not used. By default the silence in the input log mel spectrogram are ignored. + head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + expected_seq_length = self.config.max_source_positions * self.conv1.stride[0] * self.conv2.stride[0] + if input_features.shape[-1] != expected_seq_length: + raise ValueError( + f"Whisper expects the mel input features to be of length {expected_seq_length}, but found {input_features.shape[-1]}. Make sure to pad the input mel features to {expected_seq_length}." + ) + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if audio_len_after_cnn is not None: + input_mel_len = audio_len_after_cnn * 2 + max_mel_len_in_batch = int(input_mel_len.max()) + input_features = input_features[:, :, :max_mel_len_in_batch] + + inputs_embeds = nn.functional.gelu(self.conv1(input_features)) + inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds)) + + inputs_embeds = inputs_embeds.permute(0, 2, 1) + embed_pos = self.embed_positions.weight + + bsz = inputs_embeds.size(0) + src_len = inputs_embeds.size(1) + + hidden_states = inputs_embeds + embed_pos[:src_len] + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + if attention_mask is not None: + attention_mask = attention_mask.to(dtype=self.conv1.weight.dtype, + device=self.conv1.weight.device) + batch_src_len = attention_mask.size(1) + hidden_states = hidden_states[:, :batch_src_len, :] + attention_mask = attention_mask.view(bsz, -1, batch_src_len) + + attention_mask_ = attention_mask.all(1) + hidden_states[attention_mask_] = 0 + key_attention_mask = attention_mask_.view(bsz, 1, 1, batch_src_len). \ + expand(-1, self.config.encoder_attention_heads, -1, -1).reshape(bsz, self.config.encoder_attention_heads, 1, batch_src_len) + new_attention_mask = torch.zeros_like(key_attention_mask, dtype=hidden_states.dtype, device=hidden_states.device) + attention_mask = new_attention_mask.masked_fill(key_attention_mask, float("-inf")) # padding的位置attention权重置为-inf + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + # check if head_mask has a correct number of layers specified if desired + if head_mask is not None: + assert head_mask.size()[0] == ( + len(self.layers) + ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}." + + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + to_drop = False + if self.training: + dropout_probability = torch.rand([]) + if dropout_probability < self.layerdrop: # skip the layer + to_drop = True + + if to_drop: + layer_outputs = (None, None) + else: + if self.gradient_checkpointing and self.training: + # layer_outputs = self._gradient_checkpointing_func( + # encoder_layer.__call__, + # hidden_states, + # attention_mask, + # (head_mask[idx] if head_mask is not None else None), + # output_attentions, + # ) + layer_outputs = torch.utils.checkpoint.checkpoint( + encoder_layer, + hidden_states, + attention_mask, + (head_mask[idx] if head_mask is not None else None), + output_attentions, + ) + else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask=attention_mask, + layer_head_mask=(head_mask[idx] if head_mask is not None else None), + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + hidden_states = self.layer_norm(hidden_states) + if self.avg_pooler: + hidden_states = hidden_states.permute(0, 2, 1) + hidden_states = self.avg_pooler(hidden_states) + hidden_states = hidden_states.permute(0, 2, 1) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions + ) + + +class WhisperDecoder(WhisperPreTrainedModel): + """ + Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`WhisperDecoderLayer`] + + Args: + config: WhisperConfig + """ + + main_input_name = "input_ids" + + def __init__(self, config: WhisperConfig): + super().__init__(config) + self.dropout = config.dropout + self.layerdrop = config.decoder_layerdrop + self.padding_idx = config.pad_token_id + self.max_target_positions = config.max_target_positions + self.max_source_positions = config.max_source_positions + self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 + + self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, self.padding_idx) + self.embed_positions = WhisperPositionalEmbedding(self.max_target_positions, config.d_model) + + self.layers = nn.ModuleList([WhisperDecoderLayer(config) for _ in range(config.decoder_layers)]) + self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" + self._use_sdpa = config._attn_implementation == "sdpa" + + self.layer_norm = nn.LayerNorm(config.d_model) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + def forward( + self, + input_ids=None, + attention_mask=None, + encoder_hidden_states=None, + head_mask=None, + cross_attn_head_mask=None, + past_key_values=None, + inputs_embeds=None, + position_ids=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention + on hidden heads. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of + shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing + `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more + control over how to convert `input_ids` indices into associated vectors than the model's internal + embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if self._use_flash_attention_2: + # 2d mask is passed through the layers + attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None + elif self._use_sdpa and head_mask is None and not output_attentions: + # output_attentions=True & head_mask can not be supported when using SDPA. + attention_mask = _prepare_4d_causal_attention_mask_for_sdpa( + attention_mask, input_shape, inputs_embeds, past_key_values_length + ) + else: + # 4d mask is passed through the layers + attention_mask = _prepare_4d_causal_attention_mask( + attention_mask, input_shape, inputs_embeds, past_key_values_length + ) + + # embed positions + if input_ids is not None: + positions = self.embed_positions( + input_ids, past_key_values_length=past_key_values_length, position_ids=position_ids + ) + else: + positions = self.embed_positions( + inputs_embeds, past_key_values_length=past_key_values_length, position_ids=position_ids + ) + + hidden_states = inputs_embeds + positions + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..." + ) + use_cache = False + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None + next_decoder_cache = () if use_cache else None + + # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired + for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]): + if attn_mask is not None: + assert attn_mask.size()[0] == (len(self.layers)), ( + f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for" + f" {head_mask.size()[0]}." + ) + for idx, decoder_layer in enumerate(self.layers): + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + if output_hidden_states: + all_hidden_states += (hidden_states,) + if self.training: + dropout_probability = torch.rand([]) + if dropout_probability < self.layerdrop: + continue + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + attention_mask, + encoder_hidden_states, + None, # encoder attention mask + head_mask[idx] if head_mask is not None else None, + cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None, + None, # past_key_value + output_attentions, + use_cache, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + layer_head_mask=(head_mask[idx] if head_mask is not None else None), + cross_attn_layer_head_mask=( + cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None + ), + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[3 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if encoder_hidden_states is not None: + all_cross_attentions += (layer_outputs[2],) + + hidden_states = self.layer_norm(hidden_states) + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple( + v + for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions] + if v is not None + ) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + cross_attentions=all_cross_attentions, + ) + + +@add_start_docstrings( + "The bare Whisper Model outputting raw hidden-states without any specific head on top.", + WHISPER_START_DOCSTRING, +) +class WhisperModel(WhisperPreTrainedModel): + def __init__(self, config: WhisperConfig): + super().__init__(config) + + self.encoder = WhisperEncoder(config) + self.decoder = WhisperDecoder(config) + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.decoder.embed_tokens + + def set_input_embeddings(self, value): + self.decoder.embed_tokens = value + + def get_encoder(self): + return self.encoder + + def get_decoder(self): + return self.decoder + + def freeze_encoder(self): + """ + Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will + not be updated during training. + """ + self.encoder._freeze_parameters() + + def _mask_input_features( + self, + input_features: torch.FloatTensor, + attention_mask: Optional[torch.LongTensor] = None, + ): + """ + Masks extracted features along time axis and/or along feature axis according to + [SpecAugment](https://arxiv.org/abs/1904.08779). + """ + + # `config.apply_spec_augment` can set masking to False + if not getattr(self.config, "apply_spec_augment", True): + return input_features + + # generate indices & apply SpecAugment along time axis + batch_size, hidden_size, sequence_length = input_features.size() + + if self.config.mask_time_prob > 0 and self.training: + # generate indices & apply SpecAugment along time axis + mask_time_indices = _compute_mask_indices( + (batch_size, sequence_length), + mask_prob=self.config.mask_time_prob, + mask_length=self.config.mask_time_length, + attention_mask=attention_mask, + min_masks=self.config.mask_time_min_masks, + ) + mask_time_indices = torch.tensor(mask_time_indices, device=input_features.device, dtype=torch.bool) + mask_time_indices = mask_time_indices[:, None].expand(-1, hidden_size, -1) + input_features[mask_time_indices] = 0 + + if self.config.mask_feature_prob > 0 and self.training: + # generate indices & apply SpecAugment along feature axis + mask_feature_indices = _compute_mask_indices( + (batch_size, hidden_size), + mask_prob=self.config.mask_feature_prob, + mask_length=self.config.mask_feature_length, + min_masks=self.config.mask_feature_min_masks, + ) + mask_feature_indices = torch.tensor(mask_feature_indices, device=input_features.device, dtype=torch.bool) + input_features[mask_feature_indices] = 0 + + return input_features + + @add_start_docstrings_to_model_forward(WHISPER_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_features: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + decoder_inputs_embeds: Optional[Tuple[torch.FloatTensor]] = None, + decoder_position_ids: Optional[Tuple[torch.LongTensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], Seq2SeqModelOutput]: + r""" + Returns: + + Example: + ```python + >>> import torch + >>> from transformers import AutoFeatureExtractor, WhisperModel + >>> from datasets import load_dataset + + >>> model = WhisperModel.from_pretrained("openai/whisper-base") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base") + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt") + >>> input_features = inputs.input_features + >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id + >>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state + >>> list(last_hidden_state.shape) + [1, 2, 512] + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if encoder_outputs is None: + input_features = self._mask_input_features(input_features, attention_mask=attention_mask) + + encoder_outputs = self.encoder( + input_features, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, BaseModelOutput): + encoder_outputs = BaseModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + ) + + # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_outputs[0], + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=decoder_inputs_embeds, + position_ids=decoder_position_ids, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return Seq2SeqModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + +@add_start_docstrings( + "The Whisper Model with a language modeling head. Can be used for automatic speech recognition.", + WHISPER_START_DOCSTRING, +) +class WhisperForConditionalGeneration(WhisperGenerationMixin, WhisperPreTrainedModel): + base_model_prefix = "model" + _tied_weights_keys = ["proj_out.weight"] + + def __init__(self, config: WhisperConfig): + super().__init__(config) + self.model = WhisperModel(config) + self.proj_out = nn.Linear(config.d_model, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_encoder(self): + return self.model.get_encoder() + + def get_decoder(self): + return self.model.get_decoder() + + def get_output_embeddings(self): + return self.proj_out + + def set_output_embeddings(self, new_embeddings): + self.proj_out = new_embeddings + + def get_input_embeddings(self) -> nn.Module: + return self.model.get_input_embeddings() + + def freeze_encoder(self): + """ + Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will + not be updated during training. + """ + self.model.encoder._freeze_parameters() + + @add_start_docstrings_to_model_forward(WHISPER_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_features: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + decoder_inputs_embeds: Optional[Tuple[torch.FloatTensor]] = None, + decoder_position_ids: Optional[Tuple[torch.LongTensor]] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], Seq2SeqLMOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` + or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is + only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> import torch + >>> from transformers import AutoProcessor, WhisperForConditionalGeneration + >>> from datasets import load_dataset + + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") + + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + + >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt") + >>> input_features = inputs.input_features + + >>> generated_ids = model.generate(inputs=input_features) + + >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + >>> transcription + ' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.' + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if labels is not None: + if decoder_input_ids is None and decoder_inputs_embeds is None: + decoder_input_ids = shift_tokens_right( + labels, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + outputs = self.model( + input_features, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + encoder_outputs=encoder_outputs, + decoder_attention_mask=decoder_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + decoder_inputs_embeds=decoder_inputs_embeds, + decoder_position_ids=decoder_position_ids, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + lm_logits = self.proj_out(outputs[0]) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # move labels to correct device to enable PP + labels = labels.to(lm_logits.device) + loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.reshape(-1)) + + if not return_dict: + output = (lm_logits,) + outputs[1:] + return ((loss,) + output) if loss is not None else output + + return Seq2SeqLMOutput( + loss=loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + def prepare_inputs_for_generation( + self, + decoder_input_ids, + past_key_values=None, + use_cache=None, + encoder_outputs=None, + attention_mask=None, + decoder_attention_mask=None, + **kwargs, + ): + decoder_position_ids = None + if decoder_attention_mask is not None: + decoder_position_ids = (decoder_attention_mask.cumsum(-1) - 1).clamp(min=0) + + if past_key_values is not None: + past_length = past_key_values[0][0].shape[2] + + # Some generation methods already pass only the last input ID + if decoder_input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = decoder_input_ids.shape[1] - 1 + + decoder_input_ids = decoder_input_ids[:, remove_prefix_length:] + + if decoder_position_ids is not None and decoder_position_ids.shape[1] > decoder_input_ids.shape[1]: + decoder_position_ids = decoder_position_ids[:, remove_prefix_length:] + + return { + "encoder_outputs": encoder_outputs, + "past_key_values": past_key_values, + "decoder_input_ids": decoder_input_ids, + "use_cache": use_cache, + "decoder_attention_mask": decoder_attention_mask, + "decoder_position_ids": decoder_position_ids, + } + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + +class WhisperDecoderWrapper(WhisperPreTrainedModel): + """ + This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is + used in combination with the [`EncoderDecoderModel`] framework. + """ + + def __init__(self, config): + super().__init__(config) + config.is_encoder_decoder = False + self.decoder = WhisperDecoder(config) + + def get_input_embeddings(self): + return self.decoder.embed_tokens + + def set_input_embeddings(self, value): + self.decoder.embed_tokens = value + + def forward(self, *args, **kwargs): + return self.decoder(*args, **kwargs) + + +@add_start_docstrings( + """ + Whisper decoder with a language modeling head on top (linear layer with weights tied to the input embeddings). + """, + WHISPER_START_DOCSTRING, +) +class WhisperForCausalLM(WhisperPreTrainedModel): + _tied_weights_keys = ["proj_out.weight"] + main_input_name = "input_ids" + + def __init__(self, config): + super().__init__(config) + config.is_encoder_decoder = False + self.model = WhisperDecoderWrapper(config) + + self.proj_out = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.proj_out + + def set_output_embeddings(self, new_embeddings): + self.proj_out = new_embeddings + + def get_input_embeddings(self) -> nn.Module: + return self.model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.model.set_input_embeddings(value) + + def set_decoder(self, decoder): + self.model.decoder = decoder + + def get_decoder(self): + return self.model.decoder + + @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None, + head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + [What are attention masks?](../glossary#attention-mask) + encoder_outputs (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + if the model is configured as a decoder. + head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional + tensors are only required when the model is used as a decoder in a Sequence to Sequence model. Contains + pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If + `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + + Returns: + + Example: + + ```python + >>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor + >>> import torch + >>> from datasets import load_dataset + + >>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2") + >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2") + + >>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2") + + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> sample = ds[0]["audio"] + >>> input_features = processor( + ... sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt" + ... ).input_features + + >>> predicted_ids = model.generate(input_features, assistant_model=assistant_model) + + >>> # decode token ids to text + >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)[0] + >>> transcription + ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.' + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # If the user passed a tuple or `BaseModelOutput` for encoder_outputs, we extract only the hidden states + if isinstance(encoder_outputs, (BaseModelOutput, tuple, list)): + encoder_outputs = encoder_outputs[0] + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model.decoder( + input_ids=input_ids, + attention_mask=attention_mask, + encoder_hidden_states=encoder_outputs, + head_mask=head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + logits = self.proj_out(outputs[0]) + + loss = None + if labels is not None: + labels = labels.to(logits.device) + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + use_cache=None, + encoder_outputs=None, + attention_mask=None, + **kwargs, + ): + if past_key_values is not None: + past_length = past_key_values[0][0].shape[2] + + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + + input_ids = input_ids[:, remove_prefix_length:] + + return { + "encoder_outputs": encoder_outputs, + "past_key_values": past_key_values, + "input_ids": input_ids, + "use_cache": use_cache, + "attention_mask": attention_mask, + } + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + +@add_start_docstrings( + """ + Whisper Encoder Model with a sequence classification head on top (a linear layer over the pooled output) for tasks + like SUPERB Keyword Spotting. + """, + WHISPER_ENCODER_INPUTS_DOCSTRING, +) +class WhisperForAudioClassification(WhisperPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.encoder = WhisperEncoder(config) + num_layers = config.num_hidden_layers + 1 # transformer layers + input embeddings + if config.use_weighted_layer_sum: + self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers) + self.projector = nn.Linear(config.hidden_size, config.classifier_proj_size) + self.classifier = nn.Linear(config.classifier_proj_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def freeze_encoder(self): + """ + Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will + not be updated during training. Only the projection layers and classification head will be updated. + """ + self.encoder._freeze_parameters() + + def get_input_embeddings(self) -> nn.Module: + return self.encoder.get_input_embeddings() + + def set_input_embeddings(self, value: nn.Module): + self.encoder.set_input_embeddings(value) + + @add_start_docstrings_to_model_forward(WHISPER_ENCODER_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_features: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + labels: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + + Returns: + + Example: + + ```python + >>> import torch + >>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification + >>> from datasets import load_dataset + + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id") + >>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id") + + >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True) + >>> sample = next(iter(ds)) + + >>> inputs = feature_extractor( + ... sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt" + ... ) + >>> input_features = inputs.input_features + + >>> with torch.no_grad(): + ... logits = model(input_features).logits + + >>> predicted_class_ids = torch.argmax(logits).item() + >>> predicted_label = model.config.id2label[predicted_class_ids] + >>> predicted_label + 'Afrikaans' + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + if self.config.use_weighted_layer_sum: + output_hidden_states = True + elif output_hidden_states is None: + output_hidden_states = self.config.output_hidden_states + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_features, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if self.config.use_weighted_layer_sum: + hidden_states = encoder_outputs[_HIDDEN_STATES_START_POSITION] + hidden_states = torch.stack(hidden_states, dim=1) + norm_weights = nn.functional.softmax(self.layer_weights, dim=-1) + hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1) + else: + hidden_states = encoder_outputs[0] + + hidden_states = self.projector(hidden_states) + pooled_output = hidden_states.mean(dim=1) + + logits = self.classifier(pooled_output) + + loss = None + + if labels is not None: + loss_fct = CrossEntropyLoss() + # move labels to correct device to enable PP + labels = labels.to(logits.device) + loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + encoder_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutput( + loss=loss, + logits=logits, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +class WhisperOutput(ModelOutput): + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + output_lengths: Optional[torch.LongTensor] = None + + + +class AudioWhisperModel(WhisperPreTrainedModel): + """ + overwrite forward to support attention_mask + overwrite from_pretrained to support split encoder parameters from pretrained WhisperModel + """ + def __init__(self, config: WhisperConfig): + super().__init__(config) + self.encoder = WhisperEncoder(config) + + def forward( + self, + input_features, + attention_mask=None, + audio_len_after_cnn=None, + head_mask=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + + output = self.encoder( + input_features, + attention_mask, + audio_len_after_cnn, + head_mask, + output_attentions, + output_hidden_states, + return_dict + ) + + last_hidden_state = output.last_hidden_state # B x T x + return last_hidden_state + + # input_lengths = attention_mask.sum(-1) + # output_lengths = self._get_feat_extract_output_lengths(input_lengths) + # max_length = output_lengths.max() + # last_hidden_state = last_hidden_state[:,:max_length,:] + + # return WhisperOutput( + # last_hidden_state=last_hidden_state, + # hidden_states=None, + # attentions=None, + # output_lengths=output_lengths + # ) \ No newline at end of file diff --git a/internvl_chat_audio/internvl/model/audio/processing_whisper.py b/internvl_chat_audio/internvl/model/audio/processing_whisper.py new file mode 100644 index 00000000..2ebcceda --- /dev/null +++ b/internvl_chat_audio/internvl/model/audio/processing_whisper.py @@ -0,0 +1,111 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Speech processor class for Whisper +""" + + +from transformers.processing_utils import ProcessorMixin +import torch + +class WhisperProcessor(ProcessorMixin): + r""" + Constructs a Whisper processor which wraps a Whisper feature extractor and a Whisper tokenizer into a single + processor. + + [`WhisperProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`] and [`WhisperTokenizer`]. See + the [`~WhisperProcessor.__call__`] and [`~WhisperProcessor.decode`] for more information. + + Args: + feature_extractor (`WhisperFeatureExtractor`): + An instance of [`WhisperFeatureExtractor`]. The feature extractor is a required input. + tokenizer (`WhisperTokenizer`): + An instance of [`WhisperTokenizer`]. The tokenizer is a required input. + """ + attributes = ["feature_extractor"] + feature_extractor_class = "WhisperFeatureExtractor" + # tokenizer_class = "WhisperTokenizer" + + def __init__(self, feature_extractor): + super().__init__(feature_extractor) + self.current_processor = self.feature_extractor + self._in_target_context_manager = False + + def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True): + return self.tokenizer.get_decoder_prompt_ids(task=task, language=language, no_timestamps=no_timestamps) + + def get_T_after_cnn(self,L_in, dilation=1): + for (padding, kernel_size, stride) in eval("[(1,3,1)] + [(1,3,2)] "): + L_out = L_in + 2 * padding - dilation * (kernel_size - 1) - 1 + L_out = 1 + L_out // stride + L_in = L_out + return L_out + + def __call__(self, *args, **kwargs): + """ + Forwards the `audio` argument to WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] and the `text` + argument to [`~WhisperTokenizer.__call__`]. Please refer to the doctsring of the above two methods for more + information. + """ + # For backward compatibility + if self._in_target_context_manager: + return self.current_processor(*args, **kwargs) + + audio = kwargs.pop("audio", None) + sampling_rate = kwargs.pop("sampling_rate", 16000) + text = kwargs.pop("text", None) + if len(args) > 0: + audio = args[0] + args = args[1:] + + if audio is None and text is None: + raise ValueError("You need to specify either an `audio` or `text` input to process.") + + if audio is not None: + L = (audio.shape[0] if audio.shape[0] <= 480000 else 480000) # max_length < 30s + mel_len = L // 160 + audio_len_after_cnn = self.get_T_after_cnn(mel_len) + audio_token_num = (audio_len_after_cnn - 2) // 2 + 1 + inputs = self.feature_extractor(audio, *args, sampling_rate=sampling_rate, **kwargs) + inputs['audio_len_after_cnn'] = torch.tensor(audio_len_after_cnn, dtype=torch.long) + inputs['audio_token_num'] = torch.tensor(audio_token_num, dtype=torch.long) + if text is not None: + encodings = self.tokenizer(text, **kwargs) + + if text is None: + return inputs + + elif audio is None: + return encodings + else: + inputs["labels"] = encodings["input_ids"] + return inputs + + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to WhisperTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please + refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to WhisperTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to + the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + def get_prompt_ids(self, text: str, return_tensors="np"): + return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors) \ No newline at end of file diff --git a/internvl_chat_audio/internvl/model/internlm2/configuration_internlm2.py b/internvl_chat_audio/internvl/model/internlm2/configuration_internlm2.py new file mode 100644 index 00000000..282b13b1 --- /dev/null +++ b/internvl_chat_audio/internvl/model/internlm2/configuration_internlm2.py @@ -0,0 +1,150 @@ +# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved. +# +# This code is based on transformers/src/transformers/models/llama/configuration_llama.py +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" InternLM2 model configuration""" + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +INTERNLM2_PRETRAINED_CONFIG_ARCHIVE_MAP = {} + + +# Modified from transformers.model.llama.configuration_llama.LlamaConfig +class InternLM2Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternLM2Model`]. It is used to instantiate + an InternLM2 model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the InternLM2-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the InternLM2 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`InternLM2Model`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings(`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + Example: + + """ + model_type = 'internlm2' + _auto_class = 'AutoConfig' + + def __init__( # pylint: disable=W0102 + self, + vocab_size=103168, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act='silu', + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + bias=True, + rope_theta=10000, + rope_scaling=None, + attn_implementation='eager', + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.bias = bias + + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_validation() + + self.attn_implementation = attn_implementation + if self.attn_implementation is None: + self.attn_implementation = 'eager' + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2: + raise ValueError( + '`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, ' + f'got {self.rope_scaling}' + ) + rope_scaling_type = self.rope_scaling.get('type', None) + rope_scaling_factor = self.rope_scaling.get('factor', None) + if rope_scaling_type is None or rope_scaling_type not in ['linear', 'dynamic']: + raise ValueError( + f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}" + ) + if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor < 1.0: + raise ValueError(f"`rope_scaling`'s factor field must be a float >= 1, got {rope_scaling_factor}") diff --git a/internvl_chat_audio/internvl/model/internlm2/modeling_internlm2.py b/internvl_chat_audio/internvl/model/internlm2/modeling_internlm2.py new file mode 100644 index 00000000..569513df --- /dev/null +++ b/internvl_chat_audio/internvl/model/internlm2/modeling_internlm2.py @@ -0,0 +1,1429 @@ +# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved. +# +# This code is based on transformers/src/transformers/models/llama/modeling_llama.py +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch InternLM2 model.""" +import math +import queue +import threading +import warnings +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from einops import rearrange +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss +from transformers.activations import ACT2FN +from transformers.modeling_outputs import (BaseModelOutputWithPast, + CausalLMOutputWithPast, + SequenceClassifierOutputWithPast) +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import (add_start_docstrings, + add_start_docstrings_to_model_forward, logging, + replace_return_docstrings) + +try: + from transformers.generation.streamers import BaseStreamer +except: # noqa # pylint: disable=bare-except + BaseStreamer = None + +from .configuration_internlm2 import InternLM2Config + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = 'InternLM2Config' + +flash_attn_func, flash_attn_varlen_func = None, None +pad_input, index_first_axis, unpad_input = None, None, None +try: + from flash_attn import flash_attn_func as _flash_attn_func + from flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis as _index_first_axis + from flash_attn.bert_padding import pad_input as _pad_input + from flash_attn.bert_padding import unpad_input as _unpad_input + + flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func + pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input + has_flash_attn = True +except: + has_flash_attn = False + + +def _import_flash_attn(): + global flash_attn_func, flash_attn_varlen_func + global pad_input, index_first_axis, unpad_input + try: + from flash_attn import flash_attn_func as _flash_attn_func + from flash_attn import \ + flash_attn_varlen_func as _flash_attn_varlen_func + from flash_attn.bert_padding import \ + index_first_axis as _index_first_axis + from flash_attn.bert_padding import pad_input as _pad_input + from flash_attn.bert_padding import unpad_input as _unpad_input + flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func + pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input + except ImportError: + raise ImportError('flash_attn is not installed.') + + +# Copied from transformers.models.llama.modeling_llama._get_unpad_data +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +# Copied from transformers.models.bart.modeling_bart._expand_mask +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->InternLM2 +class InternLM2RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + InternLM2RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +try: + from functools import partial + + from apex.normalization import FusedRMSNorm + InternLM2RMSNorm = partial(FusedRMSNorm, eps=1e-6) # noqa + print('Discovered apex.normalization.FusedRMSNorm - will use it instead of InternLM2RMSNorm') +except ImportError: + # using the normal LlamaRMSNorm + pass +except Exception: + print('discovered apex but it failed to load, falling back to InternLM2RMSNorm') + pass + + +# Copied from transformers.model.llama.modeling_llama.LlamaRotaryEmbedding with Llama->InternLM2 +class InternLM2RotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer('inv_freq', inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype) + + freqs = torch.einsum('i,j->ij', t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False) + self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if seq_len > self.max_seq_len_cached: + self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.float32) + + return ( + self.cos_cached[:seq_len].to(dtype=x.dtype), + self.sin_cached[:seq_len].to(dtype=x.dtype), + ) + + +# Copied from transformers.model.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->InternLM2 +class InternLM2LinearScalingRotaryEmbedding(InternLM2RotaryEmbedding): + """InternLM2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev""" + + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): + self.scaling_factor = scaling_factor + super().__init__(dim, max_position_embeddings, base, device) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype) + t = t / self.scaling_factor + + freqs = torch.einsum('i,j->ij', t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False) + self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False) + + +# Copied from transformers.model.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->InternLM2 +class InternLM2DynamicNTKScalingRotaryEmbedding(InternLM2RotaryEmbedding): + """InternLM2RotaryEmbedding extended with Dynamic NTK scaling. + Credits to the Reddit users /u/bloc97 and /u/emozilla. + """ + + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): + self.scaling_factor = scaling_factor + super().__init__(dim, max_position_embeddings, base, device) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + + if seq_len > self.max_position_embeddings: + base = self.base * ( + (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1) + ) ** (self.dim / (self.dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer('inv_freq', inv_freq, persistent=False) + + t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype) + + freqs = torch.einsum('i,j->ij', t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False) + self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False) + + +# Copied from transformers.model.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2:] + return torch.cat((-x2, x1), dim=-1) + + +# Copied from transformers.model.llama.modeling_llama.apply_rotary_pos_emb +def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors.""" + cos = cos[position_ids].unsqueeze(unsqueeze_dim) + sin = sin[position_ids].unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class InternLM2MLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.w2(self.act_fn(self.w1(x)) * self.w3(x)) + + return down_proj + + +# Copied from transformers.model.llama.modeling_llama.repeat_kv +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +# Modified from transformers.model.llama.modeling_llama.LlamaAttention +class InternLM2Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: InternLM2Config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}' + f' and `num_heads`: {self.num_heads}).' + ) + + self.wqkv = nn.Linear( + self.hidden_size, + (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim, + bias=config.bias, + ) + + self.wo = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias) + self._init_rope() + + def _init_rope(self): + if self.config.rope_scaling is None: + self.rotary_emb = InternLM2RotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.config.rope_theta, + ) + else: + scaling_type = self.config.rope_scaling['type'] + scaling_factor = self.config.rope_scaling['factor'] + if scaling_type == 'dynamic': + self.rotary_emb = InternLM2DynamicNTKScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.config.rope_theta, + scaling_factor=scaling_factor, + ) + elif scaling_type == 'linear': + self.rotary_emb = InternLM2LinearScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.config.rope_theta, + scaling_factor=scaling_factor, + ) + else: + raise ValueError("Currently we only support rotary embedding's type being 'dynamic' or 'linear'.") + return self.rotary_emb + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if 'padding_mask' in kwargs: + warnings.warn( + 'Passing `padding_mask` is deprecated and will be removed in v4.37. ' + 'Please make sure use `attention_mask` instead.`' + ) + + bsz, q_len, _ = hidden_states.size() + + qkv_states = self.wqkv(hidden_states) + + qkv_states = rearrange( + qkv_states, + 'b q (h gs d) -> b q h gs d', + gs=2 + self.num_key_value_groups, + d=self.head_dim, + ) + + query_states = qkv_states[..., : self.num_key_value_groups, :] + query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d') + key_states = qkv_states[..., -2, :] + value_states = qkv_states[..., -1, :] + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f'Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is' + f' {attn_weights.size()}' + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f'Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}' + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f'`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is' + f' {attn_output.size()}' + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.wo(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +# Modified from transformers.model.llama.modeling_llama.InternLM2FlashAttention2 +class InternLM2FlashAttention2(InternLM2Attention): + """ + InternLM2 flash attention module. This module inherits from `InternLM2Attention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # InternLM2FlashAttention2 attention does not support output_attentions + if 'padding_mask' in kwargs: + warnings.warn( + 'Passing `padding_mask` is deprecated and will be removed in v4.37. ' + 'Please make sure use `attention_mask` instead.`' + ) + + # overwrite attention_mask with padding_mask + attention_mask = kwargs.pop('padding_mask') + + output_attentions = False + + bsz, q_len, _ = hidden_states.size() + + qkv_states = self.wqkv(hidden_states) + + qkv_states = rearrange( + qkv_states, + 'b q (h gs d) -> b q h gs d', + gs=2 + self.num_key_value_groups, + d=self.head_dim, + ) + + query_states = qkv_states[..., : self.num_key_value_groups, :] + query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d') + key_states = qkv_states[..., -2, :] + value_states = qkv_states[..., -1, :] + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attn_output = self._flash_attention_forward( + query_states, key_states, value_states, attention_mask, q_len + ) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.wo(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def _flash_attention_forward( + self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`int`, *optional*): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + """ + # Contains at least one padding token in the sequence + causal = self.is_causal and query_length != 1 + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._unpad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + attn_output = flash_attn_func( + query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal + ) + + return attn_output + + def _unpad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape + + key_layer = index_first_axis( + key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + value_layer = index_first_axis( + value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q.to(torch.int64), + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + + +INTERNLM2_ATTENTION_CLASSES = { + 'eager': InternLM2Attention, + 'flash_attention_2': InternLM2FlashAttention2, +} + + +# Modified from transformers.model.llama.modeling_llama.LlamaDecoderLayer +class InternLM2DecoderLayer(nn.Module): + def __init__(self, config: InternLM2Config): + super().__init__() + self.hidden_size = config.hidden_size + + self.attention = INTERNLM2_ATTENTION_CLASSES[config.attn_implementation](config=config) + + self.feed_forward = InternLM2MLP(config) + self.attention_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.ffn_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): + attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, + query_sequence_length, key_sequence_length)` if default attention is used. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + if 'padding_mask' in kwargs: + warnings.warn( + 'Passing `padding_mask` is deprecated and will be removed in v4.37. ' + 'Please make sure use `attention_mask` instead.`' + ) + + residual = hidden_states + + hidden_states = self.attention_norm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.attention( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.ffn_norm(hidden_states) + hidden_states = self.feed_forward(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +InternLM2_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`InternLM2Config`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +# Copied from transformers.models.llama.modeling_llama.LlamaPreTrainedModel with Llama->InternLM2 +@add_start_docstrings( + 'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.', + InternLM2_START_DOCSTRING, +) +class InternLM2PreTrainedModel(PreTrainedModel): + config_class = InternLM2Config + base_model_prefix = 'model' + supports_gradient_checkpointing = True + _no_split_modules = ['InternLM2DecoderLayer'] + _skip_keys_device_placement = 'past_key_values' + _supports_flash_attn_2 = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +InternLM2_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or + when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, decoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +# Modified from transformers.model.llama.modeling_llama.LlamaModel +@add_start_docstrings( + 'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.', + InternLM2_START_DOCSTRING, +) +class InternLM2Model(InternLM2PreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLM2DecoderLayer`] + + Args: + config: InternLM2Config + """ + + _auto_class = 'AutoModel' + + def __init__(self, config: InternLM2Config): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.config = config + if not has_flash_attn: + self.config.attn_implementation = 'eager' + print('Warning: Flash attention is not available, using eager attention instead.') + + self.tok_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + + self.layers = nn.ModuleList([InternLM2DecoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.tok_embeddings + + def set_input_embeddings(self, value): + self.tok_embeddings = value + + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.attn_implementation == 'flash_attention_2': + _import_flash_attn() + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time') + elif input_ids is not None: + batch_size, seq_length = input_ids.shape[:2] + elif inputs_embeds is not None: + batch_size, seq_length = inputs_embeds.shape[:2] + else: + raise ValueError('You have to specify either input_ids or inputs_embeds') + + seq_length_with_past = seq_length + past_key_values_length = 0 + if past_key_values is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0) + + if inputs_embeds is None: + inputs_embeds = self.tok_embeddings(input_ids) + + if self.config.attn_implementation == 'flash_attention_2': + # 2d mask is passed through the layers + attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None + else: + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device + ) + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length + ) + + # embed positions + hidden_states = inputs_embeds + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + + for idx, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + # None for past_key_value + return module(*inputs, output_attentions, None) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(decoder_layer), + hidden_states, + attention_mask, + position_ids, + None, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +# Modified from transformers.model.llama.modeling_llama.LlamaForCausalLM +class InternLM2ForCausalLM(InternLM2PreTrainedModel): + _auto_class = 'AutoModelForCausalLM' + + _tied_weights_keys = ['output.weight'] + + def __init__(self, config): + super().__init__(config) + self.model = InternLM2Model(config) + self.vocab_size = config.vocab_size + self.output = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.tok_embeddings + + def set_input_embeddings(self, value): + self.model.tok_embeddings = value + + def get_output_embeddings(self): + return self.output + + def set_output_embeddings(self, new_embeddings): + self.output = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, InternLM2ForCausalLM + + >>> model = InternLM2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS) + >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER) + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.output(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + device = input_ids.device if input_ids is not None else inputs_embeds.device + output = CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + output['logits'] = output['logits'].to(device) + return output + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + if past_key_values is not None: + past_length = past_key_values[0][0].shape[2] + + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + + input_ids = input_ids[:, remove_prefix_length:] + + position_ids = kwargs.get('position_ids', None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1]:] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {'inputs_embeds': inputs_embeds} + else: + model_inputs = {'input_ids': input_ids} + + model_inputs.update( + { + 'position_ids': position_ids, + 'past_key_values': past_key_values, + 'use_cache': kwargs.get('use_cache'), + 'attention_mask': attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = [], meta_instruction=''): + if tokenizer.add_bos_token: + prompt = '' + else: + prompt = tokenizer.bos_token + if meta_instruction: + prompt += f"""<|im_start|>system\n{meta_instruction}<|im_end|>\n""" + for record in history: + prompt += f"""<|im_start|>user\n{record[0]}<|im_end|>\n<|im_start|>assistant\n{record[1]}<|im_end|>\n""" + prompt += f"""<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant\n""" + return tokenizer([prompt], return_tensors='pt') + + @torch.no_grad() + def chat( + self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = [], + streamer: Optional[BaseStreamer] = None, + max_new_tokens: int = 1024, + do_sample: bool = True, + temperature: float = 0.8, + top_p: float = 0.8, + meta_instruction: str = 'You are an AI assistant whose name is InternLM (书生·浦语).\n' + '- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n' + '- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.', + **kwargs, + ): + inputs = self.build_inputs(tokenizer, query, history, meta_instruction) + inputs = {k: v.to(self.device) for k, v in inputs.items() if torch.is_tensor(v)} + # also add end-of-assistant token in eos token id to avoid unnecessary generation + eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids(['<|im_end|>'])[0]] + outputs = self.generate( + **inputs, + streamer=streamer, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + temperature=temperature, + top_p=top_p, + eos_token_id=eos_token_id, + **kwargs, + ) + outputs = outputs[0].cpu().tolist()[len(inputs['input_ids'][0]):] + response = tokenizer.decode(outputs, skip_special_tokens=True) + response = response.split('<|im_end|>')[0] + history = history + [(query, response)] + return response, history + + @torch.no_grad() + def stream_chat( + self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = [], + max_new_tokens: int = 1024, + do_sample: bool = True, + temperature: float = 0.8, + top_p: float = 0.8, + **kwargs, + ): + """ + Return a generator in format: (response, history) + Eg. + ('你好，有什么可以帮助您的吗', [('你好', '你好，有什么可以帮助您的吗')]) + ('你好，有什么可以帮助您的吗？', [('你好', '你好，有什么可以帮助您的吗？')]) + """ + if BaseStreamer is None: + raise ModuleNotFoundError( + 'The version of `transformers` is too low. Please make sure ' + 'that you have installed `transformers>=4.28.0`.' + ) + + response_queue = queue.Queue(maxsize=20) + + class ChatStreamer(BaseStreamer): + def __init__(self, tokenizer) -> None: + super().__init__() + self.tokenizer = tokenizer + self.queue = response_queue + self.query = query + self.history = history + self.response = '' + self.cache = [] + self.received_inputs = False + self.queue.put((self.response, history + [(self.query, self.response)])) + + def put(self, value): + if len(value.shape) > 1 and value.shape[0] > 1: + raise ValueError('ChatStreamer only supports batch size 1') + elif len(value.shape) > 1: + value = value[0] + + if not self.received_inputs: + # The first received value is input_ids, ignore here + self.received_inputs = True + return + + self.cache.extend(value.tolist()) + token = self.tokenizer.decode(self.cache, skip_special_tokens=True) + if token.strip() != '<|im_end|>': + self.response = self.response + token + history = self.history + [(self.query, self.response)] + self.queue.put((self.response, history)) + self.cache = [] + else: + self.end() + + def end(self): + self.queue.put(None) + + def stream_producer(): + return self.chat( + tokenizer=tokenizer, + query=query, + streamer=ChatStreamer(tokenizer=tokenizer), + history=history, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + temperature=temperature, + top_p=top_p, + **kwargs, + ) + + def consumer(): + producer = threading.Thread(target=stream_producer) + producer.start() + while True: + res = response_queue.get() + if res is None: + return + yield res + + return consumer() + + +# Copied from transformers.model.llama.modeling_llama.LlamaForSequenceClassification with Llama->InternLM2 +@add_start_docstrings( + """ + The InternLM2 Model transformer with a sequence classification head on top (linear layer). + + [`InternLM2ForSequenceClassification`] uses the last token in order to do the classification, + as other causal models (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + InternLM2_START_DOCSTRING, +) +class InternLM2ForSequenceClassification(InternLM2PreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = InternLM2Model(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.tok_embeddings + + def set_input_embeddings(self, value): + self.model.tok_embeddings = value + + @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError('Cannot handle batch sizes > 1 if no padding token is defined.') + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + sequence_lengths = (torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1).to( + logits.device + ) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + labels = labels.to(logits.device) + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = 'regression' + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = 'single_label_classification' + else: + self.config.problem_type = 'multi_label_classification' + + if self.config.problem_type == 'regression': + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == 'single_label_classification': + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == 'multi_label_classification': + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/internvl_chat_audio/internvl/model/internlm2/tokenization_internlm2.py b/internvl_chat_audio/internvl/model/internlm2/tokenization_internlm2.py new file mode 100644 index 00000000..1be581da --- /dev/null +++ b/internvl_chat_audio/internvl/model/internlm2/tokenization_internlm2.py @@ -0,0 +1,235 @@ +# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved. +# +# This code is based on transformers/src/transformers/models/llama/tokenization_llama.py +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization classes for InternLM.""" +import os +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple + +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': './tokenizer.model'} + +PRETRAINED_VOCAB_FILES_MAP = {} + + +# Modified from transformers.model.llama.tokenization_llama.LlamaTokenizer +class InternLM2Tokenizer(PreTrainedTokenizer): + """ + Construct a InternLM2 tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + model_input_names = ['input_ids', 'attention_mask'] + _auto_class = 'AutoTokenizer' + + def __init__( + self, + vocab_file, + unk_token='', + bos_token='', + eos_token='', + pad_token='', + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + decode_with_prefix_space=False, + clean_up_tokenization_spaces=False, + **kwargs, + ): + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + self.vocab_file = vocab_file + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + self.decode_with_prefix_space = decode_with_prefix_space + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(vocab_file) + self._no_prefix_space_tokens = None + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + + @property + def no_prefix_space_tokens(self): + if self._no_prefix_space_tokens is None: + vocab = self.convert_ids_to_tokens(list(range(self.vocab_size))) + self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith('▁')} + return self._no_prefix_space_tokens + + @property + def vocab_size(self): + """Returns vocab size""" + return self.sp_model.get_piece_size() + + @property + def bos_token_id(self) -> Optional[int]: + return self.sp_model.bos_id() + + @property + def eos_token_id(self) -> Optional[int]: + return self.sp_model.eos_id() + + def get_vocab(self): + """Returns vocab as a dict""" + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text): + """Returns a tokenized string.""" + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + token = self.sp_model.IdToPiece(index) + return token + + def _maybe_add_prefix_space(self, tokens, decoded): + if tokens and tokens[0] not in self.no_prefix_space_tokens: + return ' ' + decoded + else: + return decoded + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + current_sub_tokens = [] + out_string = '' + prev_is_special = False + for token in tokens: + # make sure that special tokens are not decoded using sentencepiece model + if token in self.all_special_tokens: + if not prev_is_special: + out_string += ' ' + out_string += self.sp_model.decode(current_sub_tokens) + token + prev_is_special = True + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + prev_is_special = False + out_string += self.sp_model.decode(current_sub_tokens) + out_string = self.clean_up_tokenization(out_string) + out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string) + return out_string[1:] + + def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]: + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + logger.error(f'Vocabulary path ({save_directory}) should be a directory') + return + out_vocab_file = os.path.join( + save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, 'wb') as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + if self.add_bos_token: + bos_token_ids = [self.bos_token_id] + else: + bos_token_ids = [] + + output = bos_token_ids + token_ids_0 + + if token_ids_1 is not None: + output = output + token_ids_1 + + if self.add_eos_token: + output = output + [self.eos_token_id] + + return output + + def get_special_tokens_mask( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + ) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make + use of token type ids, therefore a list of zeros is returned. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of zeros. + """ + eos = [self.eos_token_id] + + if token_ids_1 is None: + return len(token_ids_0 + eos) * [0] + return len(token_ids_0 + eos + token_ids_1 + eos) * [0] diff --git a/internvl_chat_audio/internvl/model/internlm2/tokenization_internlm2_fast.py b/internvl_chat_audio/internvl/model/internlm2/tokenization_internlm2_fast.py new file mode 100644 index 00000000..aa0fccbd --- /dev/null +++ b/internvl_chat_audio/internvl/model/internlm2/tokenization_internlm2_fast.py @@ -0,0 +1,211 @@ +# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved. +# +# This code is based on transformers/src/transformers/models/llama/tokenization_llama_fast.py +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization Fast class for InternLM.""" +import os +from shutil import copyfile +from typing import Any, Dict, Optional, Tuple + +from tokenizers import Tokenizer, decoders, normalizers, processors +from tokenizers.models import BPE +from transformers.convert_slow_tokenizer import (SLOW_TO_FAST_CONVERTERS, + SentencePieceExtractor, + SpmConverter) +from transformers.tokenization_utils_fast import PreTrainedTokenizerFast +from transformers.utils import logging + +from .tokenization_internlm2 import InternLM2Tokenizer + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': './tokenizer.model'} + + +# Modified from transformers.convert_slow_tokenizer.LlamaConverter +class InternLM2Converter(SpmConverter): + handle_byte_fallback = True + + def vocab(self, proto): + vocab = [ + ('', 0.0), + ('', 0.0), + ('', 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + return vocab + + def unk_id(self, proto): + unk_id = 0 + return unk_id + + def decoder(self, replacement, add_prefix_space): + return decoders.Sequence( + [ + decoders.Replace('▁', ' '), + decoders.ByteFallback(), + decoders.Fuse(), + decoders.Strip(content=' ', left=1), + ] + ) + + def tokenizer(self, proto): + model_type = proto.trainer_spec.model_type + vocab_scores = self.vocab(proto) + # special tokens + added_tokens = self.original_tokenizer.added_tokens_decoder + for i in range(len(vocab_scores)): + piece, score = vocab_scores[i] + if i in added_tokens: + vocab_scores[i] = (added_tokens[i].content, score) + if model_type == 1: + raise RuntimeError('InternLM2 is supposed to be a BPE model!') + + elif model_type == 2: + _, merges = SentencePieceExtractor(self.original_tokenizer.vocab_file).extract(vocab_scores) + bpe_vocab = {word: i for i, (word, _score) in enumerate(vocab_scores)} + tokenizer = Tokenizer( + BPE(bpe_vocab, merges, unk_token=proto.trainer_spec.unk_piece, fuse_unk=True, byte_fallback=True) + ) + tokenizer.add_special_tokens( + [ added_token for index, added_token in added_tokens.items()] + ) + else: + raise Exception( + "You're trying to run a `Unigram` model but you're file was trained with a different algorithm" + ) + + return tokenizer + + def normalizer(self, proto): + normalizers_list = [] + if proto.normalizer_spec.add_dummy_prefix: + normalizers_list.append(normalizers.Prepend(prepend='▁')) + normalizers_list.append(normalizers.Replace(pattern=' ', content='▁')) + return normalizers.Sequence(normalizers_list) + + def pre_tokenizer(self, replacement, add_prefix_space): + return None + + +SLOW_TO_FAST_CONVERTERS['InternLM2Tokenizer'] = InternLM2Converter + + +# Modified from transformers.model.llama.tokenization_llama_fast.LlamaTokenizerFast -> InternLM2TokenizerFast +class InternLM2TokenizerFast(PreTrainedTokenizerFast): + vocab_files_names = VOCAB_FILES_NAMES + slow_tokenizer_class = InternLM2Tokenizer + padding_side = 'left' + model_input_names = ['input_ids', 'attention_mask'] + _auto_class = 'AutoTokenizer' + + def __init__( + self, + vocab_file, + unk_token='', + bos_token='', + eos_token='', + pad_token='', + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + decode_with_prefix_space=False, + clean_up_tokenization_spaces=False, + **kwargs, + ): + super().__init__( + vocab_file=vocab_file, + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + sp_model_kwargs=sp_model_kwargs, + add_bos_token=add_bos_token, + add_eos_token=add_eos_token, + decode_with_prefix_space=decode_with_prefix_space, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + self._add_bos_token = add_bos_token + self._add_eos_token = add_eos_token + self.update_post_processor() + self.vocab_file = vocab_file + + @property + def can_save_slow_tokenizer(self) -> bool: + return os.path.isfile(self.vocab_file) if self.vocab_file else False + + def update_post_processor(self): + """ + Updates the underlying post processor with the current `bos_token` and `eos_token`. + """ + bos = self.bos_token + bos_token_id = self.bos_token_id + if bos is None and self.add_bos_token: + raise ValueError('add_bos_token = True but bos_token = None') + + eos = self.eos_token + eos_token_id = self.eos_token_id + if eos is None and self.add_eos_token: + raise ValueError('add_eos_token = True but eos_token = None') + + single = f"{(bos+':0 ') if self.add_bos_token else ''}$A:0{(' '+eos+':0') if self.add_eos_token else ''}" + pair = f"{single}{(' '+bos+':1') if self.add_bos_token else ''} $B:1{(' '+eos+':1') if self.add_eos_token else ''}" + + special_tokens = [] + if self.add_bos_token: + special_tokens.append((bos, bos_token_id)) + if self.add_eos_token: + special_tokens.append((eos, eos_token_id)) + self._tokenizer.post_processor = processors.TemplateProcessing( + single=single, pair=pair, special_tokens=special_tokens + ) + + @property + def add_eos_token(self): + return self._add_eos_token + + @property + def add_bos_token(self): + return self._add_bos_token + + @add_eos_token.setter + def add_eos_token(self, value): + self._add_eos_token = value + self.update_post_processor() + + @add_bos_token.setter + def add_bos_token(self, value): + self._add_bos_token = value + self.update_post_processor() + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + if not self.can_save_slow_tokenizer: + raise ValueError( + 'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow ' + 'tokenizer.' + ) + + if not os.path.isdir(save_directory): + logger.error(f'Vocabulary path ({save_directory}) should be a directory') + return + out_vocab_file = os.path.join( + save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) diff --git a/internvl_chat_audio/internvl/model/internvl_chat/__init__.py b/internvl_chat_audio/internvl/model/internvl_chat/__init__.py new file mode 100644 index 00000000..55d67377 --- /dev/null +++ b/internvl_chat_audio/internvl/model/internvl_chat/__init__.py @@ -0,0 +1,17 @@ +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .configuration_intern_vit import InternVisionConfig +from .configuration_internvl_chat import InternVLChatConfig +from .configuration_internvl_audio_chat import InternVLChatAudioConfig +from .modeling_intern_vit import InternVisionModel +from .modeling_internvl_chat import InternVLChatModel +from .modeling_internvl_audio import InternVLChatAudioModel + +__all__ = ['InternVisionConfig', 'InternVisionModel', + 'InternVLChatConfig', 'InternVLChatModel', + 'InternVLChatAudioConfig', 'InternVLChatAudioModel', + ] diff --git a/internvl_chat_audio/internvl/model/internvl_chat/configuration_intern_vit.py b/internvl_chat_audio/internvl/model/internvl_chat/configuration_intern_vit.py new file mode 100644 index 00000000..ac60112c --- /dev/null +++ b/internvl_chat_audio/internvl/model/internvl_chat/configuration_intern_vit.py @@ -0,0 +1,119 @@ +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +import os +from typing import Union + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +class InternVisionConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternVisionModel`]. It is used to + instantiate a vision encoder according to the specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + num_channels (`int`, *optional*, defaults to 3): + Number of color channels in the input images (e.g., 3 for RGB). + patch_size (`int`, *optional*, defaults to 14): + The size (resolution) of each patch. + image_size (`int`, *optional*, defaults to 224): + The size (resolution) of each image. + qkv_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries and values in the self-attention layers. + hidden_size (`int`, *optional*, defaults to 3200): + Dimensionality of the encoder layers and the pooler layer. + num_attention_heads (`int`, *optional*, defaults to 25): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 12800): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + qk_normalization (`bool`, *optional*, defaults to `True`): + Whether to normalize the queries and keys in the self-attention layers. + num_hidden_layers (`int`, *optional*, defaults to 48): + Number of hidden layers in the Transformer encoder. + use_flash_attn (`bool`, *optional*, defaults to `True`): + Whether to use flash attention mechanism. + hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. + layer_norm_eps (`float`, *optional*, defaults to 1e-6): + The epsilon used by the layer normalization layers. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + drop_path_rate (`float`, *optional*, defaults to 0.0): + Dropout rate for stochastic depth. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float`, *optional*, defaults to 0.1): + A factor for layer scale. + """ + + model_type = 'intern_vit_6b' + + def __init__( + self, + num_channels=3, + patch_size=14, + image_size=224, + qkv_bias=False, + hidden_size=3200, + num_attention_heads=25, + intermediate_size=12800, + qk_normalization=True, + num_hidden_layers=48, + use_flash_attn=True, + hidden_act='gelu', + norm_type='rms_norm', + layer_norm_eps=1e-6, + dropout=0.0, + drop_path_rate=0.0, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=0.1, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.dropout = dropout + self.drop_path_rate = drop_path_rate + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + self.norm_type = norm_type + self.qkv_bias = qkv_bias + self.qk_normalization = qk_normalization + self.use_flash_attn = use_flash_attn + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig': + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + + if 'vision_config' in config_dict: + config_dict = config_dict['vision_config'] + + if 'model_type' in config_dict and hasattr(cls, 'model_type') and config_dict['model_type'] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.' + ) + + return cls.from_dict(config_dict, **kwargs) diff --git a/internvl_chat_audio/internvl/model/internvl_chat/configuration_internvl_audio_chat.py b/internvl_chat_audio/internvl/model/internvl_chat/configuration_internvl_audio_chat.py new file mode 100644 index 00000000..a35e411d --- /dev/null +++ b/internvl_chat_audio/internvl/model/internvl_chat/configuration_internvl_audio_chat.py @@ -0,0 +1,50 @@ +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import copy + +from internvl.model.internlm2.configuration_internlm2 import InternLM2Config +from internvl.model.phi3.configuration_phi3 import Phi3Config +from transformers import AutoConfig, LlamaConfig, Qwen2Config +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging +from internvl.model.audio.configuration_whisper import WhisperConfig + +from .configuration_intern_vit import InternVisionConfig +from .configuration_internvl_chat import InternVLChatConfig + +logger = logging.get_logger(__name__) + + + + +class InternVLChatAudioConfig(InternVLChatConfig): + model_type = "internvl_chat" + is_composition = True + + def __init__( + self, + vision_config=None, + audio_config=None, + llm_config=None, + **kwargs): + super().__init__(vision_config, llm_config, **kwargs) + + if audio_config is None: + audio_config = {} + logger.info('audio_config is None. Initializing the Audioconfig with default values.') + self.audio_config = WhisperConfig(**audio_config) + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = super().to_dict() + output['audio_config'] = self.audio_config.to_dict() + return output \ No newline at end of file diff --git a/internvl_chat_audio/internvl/model/internvl_chat/configuration_internvl_chat.py b/internvl_chat_audio/internvl/model/internvl_chat/configuration_internvl_chat.py new file mode 100644 index 00000000..8466a16a --- /dev/null +++ b/internvl_chat_audio/internvl/model/internvl_chat/configuration_internvl_chat.py @@ -0,0 +1,104 @@ +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import copy + +from internvl.model.internlm2.configuration_internlm2 import InternLM2Config +from internvl.model.phi3.configuration_phi3 import Phi3Config +from transformers import AutoConfig, LlamaConfig, Qwen2Config +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +from .configuration_intern_vit import InternVisionConfig + +logger = logging.get_logger(__name__) + + +class InternVLChatConfig(PretrainedConfig): + model_type = 'internvl_chat' + is_composition = True + + def __init__( + self, + vision_config=None, + llm_config=None, + use_backbone_lora=0, + use_llm_lora=0, + pad2square=False, + select_layer=-1, + force_image_size=None, + downsample_ratio=0.5, + template=None, + dynamic_image_size=False, + use_thumbnail=False, + ps_version='v1', + min_dynamic_patch=1, + max_dynamic_patch=6, + **kwargs): + super().__init__(**kwargs) + + if vision_config is None: + vision_config = {} + logger.info('vision_config is None. Initializing the InternVisionConfig with default values.') + + if llm_config is None: + llm_config = {} + logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).') + + self.vision_config = InternVisionConfig(**vision_config) + if llm_config['architectures'][0] == 'LlamaForCausalLM': + self.llm_config = LlamaConfig(**llm_config) + elif llm_config['architectures'][0] == 'InternLM2ForCausalLM': + self.llm_config = InternLM2Config(**llm_config) + elif llm_config['architectures'][0] == 'Phi3ForCausalLM': + self.llm_config = Phi3Config(**llm_config) + elif llm_config['architectures'][0] == 'Qwen2ForCausalLM': + self.llm_config = Qwen2Config(**llm_config) + else: + raise ValueError('Unsupported architecture: {}'.format(llm_config['architectures'][0])) + self.use_backbone_lora = use_backbone_lora + self.use_llm_lora = use_llm_lora + self.pad2square = pad2square + self.select_layer = select_layer + self.force_image_size = force_image_size + self.downsample_ratio = downsample_ratio + self.template = template + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.ps_version = ps_version # pixel shuffle version + self.min_dynamic_patch = min_dynamic_patch + self.max_dynamic_patch = max_dynamic_patch + + logger.info(f'vision_select_layer: {self.select_layer}') + logger.info(f'ps_version: {self.ps_version}') + logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}') + logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}') + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output['vision_config'] = self.vision_config.to_dict() + output['llm_config'] = self.llm_config.to_dict() + output['model_type'] = self.__class__.model_type + output['use_backbone_lora'] = self.use_backbone_lora + output['use_llm_lora'] = self.use_llm_lora + output['pad2square'] = self.pad2square + output['select_layer'] = self.select_layer + output['force_image_size'] = self.force_image_size + output['downsample_ratio'] = self.downsample_ratio + output['template'] = self.template + output['dynamic_image_size'] = self.dynamic_image_size + output['use_thumbnail'] = self.use_thumbnail + output['ps_version'] = self.ps_version + output['min_dynamic_patch'] = self.min_dynamic_patch + output['max_dynamic_patch'] = self.max_dynamic_patch + + return output diff --git a/internvl_chat_audio/internvl/model/internvl_chat/flash_attention.py b/internvl_chat_audio/internvl/model/internvl_chat/flash_attention.py new file mode 100644 index 00000000..7cda9bfa --- /dev/null +++ b/internvl_chat_audio/internvl/model/internvl_chat/flash_attention.py @@ -0,0 +1,76 @@ +# https://github.com/Dao-AILab/flash-attention/blob/v0.2.8/flash_attn/flash_attention.py +import torch +import torch.nn as nn +from einops import rearrange + +try: # v1 + from flash_attn.flash_attn_interface import \ + flash_attn_unpadded_qkvpacked_func +except: # v2 + from flash_attn.flash_attn_interface import flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func + +from flash_attn.bert_padding import pad_input, unpad_input + + +class FlashAttention(nn.Module): + """Implement the scaled dot product attention with softmax. + Arguments + --------- + softmax_scale: The temperature to use for the softmax attention. + (default: 1/sqrt(d_keys) where d_keys is computed at + runtime) + attention_dropout: The dropout rate to apply to the attention + (default: 0.0) + """ + + def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None): + super().__init__() + self.softmax_scale = softmax_scale + self.dropout_p = attention_dropout + + def forward(self, qkv, key_padding_mask=None, causal=False, cu_seqlens=None, + max_s=None, need_weights=False): + """Implements the multihead softmax attention. + Arguments + --------- + qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) if key_padding_mask is None + if unpadded: (nnz, 3, h, d) + key_padding_mask: a bool tensor of shape (B, S) + """ + assert not need_weights + assert qkv.dtype in [torch.float16, torch.bfloat16] + assert qkv.is_cuda + + if cu_seqlens is None: + batch_size = qkv.shape[0] + seqlen = qkv.shape[1] + if key_padding_mask is None: + qkv = rearrange(qkv, 'b s ... -> (b s) ...') + max_s = seqlen + cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32, + device=qkv.device) + output = flash_attn_unpadded_qkvpacked_func( + qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0, + softmax_scale=self.softmax_scale, causal=causal + ) + output = rearrange(output, '(b s) ... -> b s ...', b=batch_size) + else: + nheads = qkv.shape[-2] + x = rearrange(qkv, 'b s three h d -> b s (three h d)') + x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask) + x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads) + output_unpad = flash_attn_unpadded_qkvpacked_func( + x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0, + softmax_scale=self.softmax_scale, causal=causal + ) + output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'), + indices, batch_size, seqlen), + 'b s (h d) -> b s h d', h=nheads) + else: + assert max_s is not None + output = flash_attn_unpadded_qkvpacked_func( + qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0, + softmax_scale=self.softmax_scale, causal=causal + ) + + return output, None diff --git a/internvl_chat_audio/internvl/model/internvl_chat/modeling_intern_vit.py b/internvl_chat_audio/internvl/model/internvl_chat/modeling_intern_vit.py new file mode 100644 index 00000000..5d875f4b --- /dev/null +++ b/internvl_chat_audio/internvl/model/internvl_chat/modeling_intern_vit.py @@ -0,0 +1,362 @@ +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from typing import Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from einops import rearrange +from timm.models.layers import DropPath +from torch import nn +from transformers.activations import ACT2FN +from transformers.modeling_outputs import (BaseModelOutput, + BaseModelOutputWithPooling) +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import logging + +from .configuration_intern_vit import InternVisionConfig + +try: + from .flash_attention import FlashAttention + has_flash_attn = True +except: + print('FlashAttention is not installed.') + has_flash_attn = False + +logger = logging.get_logger(__name__) + + +class InternRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +try: + from apex.normalization import FusedRMSNorm + + InternRMSNorm = FusedRMSNorm # noqa + + logger.info('Discovered apex.normalization.FusedRMSNorm - will use it instead of InternRMSNorm') +except ImportError: + # using the normal InternRMSNorm + pass +except Exception: + logger.warning('discovered apex but it failed to load, falling back to InternRMSNorm') + pass + + +NORM2FN = { + 'rms_norm': InternRMSNorm, + 'layer_norm': nn.LayerNorm, +} + + +class InternVisionEmbeddings(nn.Module): + def __init__(self, config: InternVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter( + torch.randn(1, 1, self.embed_dim), + ) + + self.patch_embedding = nn.Conv2d( + in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size + ) + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + 1 + + self.position_embedding = nn.Parameter(torch.randn(1, self.num_positions, self.embed_dim)) + + def _get_pos_embed(self, pos_embed, H, W): + target_dtype = pos_embed.dtype + pos_embed = pos_embed.float().reshape( + 1, self.image_size // self.patch_size, self.image_size // self.patch_size, -1).permute(0, 3, 1, 2) + pos_embed = F.interpolate(pos_embed, size=(H, W), mode='bicubic', align_corners=False). \ + reshape(1, -1, H * W).permute(0, 2, 1).to(target_dtype) + return pos_embed + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values) # shape = [*, channel, width, height] + batch_size, _, height, width = patch_embeds.shape + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + class_embeds = self.class_embedding.expand(batch_size, 1, -1).to(target_dtype) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + position_embedding = torch.cat([ + self.position_embedding[:, :1, :], + self._get_pos_embed(self.position_embedding[:, 1:, :], height, width) + ], dim=1) + embeddings = embeddings + position_embedding.to(target_dtype) + return embeddings + + +class InternAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: InternVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.use_flash_attn = config.use_flash_attn and has_flash_attn + if config.use_flash_attn and not has_flash_attn: + print('Warning: Flash Attention is not available, use_flash_attn is set to False.') + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f'embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:' + f' {self.num_heads}).' + ) + + self.scale = self.head_dim ** -0.5 + self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim, bias=config.qkv_bias) + self.attn_drop = nn.Dropout(config.attention_dropout) + self.proj_drop = nn.Dropout(config.dropout) + + self.qk_normalization = config.qk_normalization + + if self.qk_normalization: + self.q_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps) + self.k_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps) + + if self.use_flash_attn: + self.inner_attn = FlashAttention(attention_dropout=config.attention_dropout) + self.proj = nn.Linear(self.embed_dim, self.embed_dim) + + def _naive_attn(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + if self.qk_normalization: + B_, H_, N_, D_ = q.shape + q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2) + k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2) + + attn = ((q * self.scale) @ k.transpose(-2, -1)) + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + def _flash_attn(self, x, key_padding_mask=None, need_weights=False): + qkv = self.qkv(x) + qkv = rearrange(qkv, 'b s (three h d) -> b s three h d', three=3, h=self.num_heads) + + if self.qk_normalization: + q, k, v = qkv.unbind(2) + q = self.q_norm(q.flatten(-2, -1)).view(q.shape) + k = self.k_norm(k.flatten(-2, -1)).view(k.shape) + qkv = torch.stack([q, k, v], dim=2) + + context, _ = self.inner_attn( + qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=False + ) + outs = self.proj(rearrange(context, 'b s h d -> b s (h d)')) + outs = self.proj_drop(outs) + return outs + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + x = self._naive_attn(hidden_states) if not self.use_flash_attn else self._flash_attn(hidden_states) + return x + + +class InternMLP(nn.Module): + def __init__(self, config: InternVisionConfig): + super().__init__() + self.config = config + self.act = ACT2FN[config.hidden_act] + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +class InternVisionEncoderLayer(nn.Module): + def __init__(self, config: InternVisionConfig, drop_path_rate: float): + super().__init__() + self.embed_dim = config.hidden_size + self.intermediate_size = config.intermediate_size + self.norm_type = config.norm_type + + self.attn = InternAttention(config) + self.mlp = InternMLP(config) + self.norm1 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps) + self.norm2 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps) + + self.ls1 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim)) + self.ls2 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim)) + self.drop_path1 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity() + self.drop_path2 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity() + + def forward( + self, + hidden_states: torch.Tensor, + ) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor], Optional[Tuple[torch.FloatTensor]]]: + """ + Args: + hidden_states (`Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]`): input to the layer of shape `(batch, seq_len, embed_dim)` + """ + hidden_states = hidden_states + self.drop_path1(self.attn(self.norm1(hidden_states)) * self.ls1) + + hidden_states = hidden_states + self.drop_path2(self.mlp(self.norm2(hidden_states)) * self.ls2) + + return hidden_states + + +class InternVisionEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a + [`InternEncoderLayer`]. + + Args: + config (`InternConfig`): + The corresponding vision configuration for the `InternEncoder`. + """ + + def __init__(self, config: InternVisionConfig): + super().__init__() + self.config = config + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers)] + self.layers = nn.ModuleList([ + InternVisionEncoderLayer(config, dpr[idx]) for idx in range(config.num_hidden_layers)]) + self.gradient_checkpointing = True + + def forward( + self, + inputs_embeds, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Embedded representation of the inputs. Should be float, not int tokens. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + encoder_states = () if output_hidden_states else None + hidden_states = inputs_embeds + + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + if self.gradient_checkpointing and self.training: + layer_outputs = torch.utils.checkpoint.checkpoint( + encoder_layer, + hidden_states) + else: + layer_outputs = encoder_layer( + hidden_states, + ) + hidden_states = layer_outputs + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, encoder_states] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, hidden_states=encoder_states + ) + + +class InternVisionModel(PreTrainedModel): + main_input_name = 'pixel_values' + config_class = InternVisionConfig + _no_split_modules = ['InternVisionEncoderLayer'] + + def __init__(self, config: InternVisionConfig): + super().__init__(config) + self.config = config + + self.embeddings = InternVisionEmbeddings(config) + self.encoder = InternVisionEncoder(config) + + def resize_pos_embeddings(self, old_size, new_size, patch_size): + pos_emb = self.embeddings.position_embedding + _, num_positions, embed_dim = pos_emb.shape + cls_emb = pos_emb[:, :1, :] + pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size, old_size // patch_size, -1).permute(0, 3, 1, 2) + pos_emb = F.interpolate(pos_emb.float(), size=new_size // patch_size, mode='bicubic', align_corners=False) + pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim, -1).permute(0, 2, 1) + pos_emb = torch.cat([cls_emb, pos_emb], dim=1) + self.embeddings.position_embedding = nn.Parameter(pos_emb) + self.embeddings.image_size = new_size + logger.info('Resized position embeddings from {} to {}'.format(old_size, new_size)) + + def get_input_embeddings(self): + return self.embeddings + + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + pixel_embeds: Optional[torch.FloatTensor] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None and pixel_embeds is None: + raise ValueError('You have to specify pixel_values or pixel_embeds') + + if pixel_embeds is not None: + hidden_states = pixel_embeds + else: + if len(pixel_values.shape) == 4: + hidden_states = self.embeddings(pixel_values) + else: + raise ValueError(f'wrong pixel_values size: {pixel_values.shape}') + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + last_hidden_state = encoder_outputs.last_hidden_state + pooled_output = last_hidden_state[:, 0, :] + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) diff --git a/internvl_chat_audio/internvl/model/internvl_chat/modeling_internvl_audio.py b/internvl_chat_audio/internvl/model/internvl_chat/modeling_internvl_audio.py new file mode 100644 index 00000000..0fc05e76 --- /dev/null +++ b/internvl_chat_audio/internvl/model/internvl_chat/modeling_internvl_audio.py @@ -0,0 +1,622 @@ +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +import warnings +from typing import Any, List, Optional, Tuple, Union +from PIL import Image, ImageDraw +from io import BytesIO +import requests +import torch.distributed as dist +import torch.utils.checkpoint +from internvl.model.internlm2.modeling_internlm2 import InternLM2ForCausalLM +from internvl.model.phi3.modeling_phi3 import Phi3ForCausalLM +from peft import LoraConfig, get_peft_model +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers import (AutoModel, GenerationConfig, LlamaForCausalLM, + LlamaTokenizer, Qwen2ForCausalLM) +from transformers.modeling_outputs import CausalLMOutputWithPast +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import ModelOutput, logging +from internvl.conversation import get_conv_template +from .configuration_internvl_chat import InternVLChatConfig +from .modeling_intern_vit import InternVisionModel +from .modeling_internvl_chat import InternVLChatModel +from .configuration_internvl_audio_chat import InternVLChatAudioConfig +from internvl.model.audio.modeling_whisper import AudioWhisperModel +from internvl.conversation import get_conv_template +from internvl.train.dataset_audio import * +from internvl.train.dataset import dynamic_preprocess +from internvl.train.constants import (BOX_END_TOKEN, BOX_START_TOKEN, + IMG_CONTEXT_TOKEN, IMG_END_TOKEN, + IMG_START_TOKEN, QUAD_END_TOKEN, + QUAD_START_TOKEN, REF_END_TOKEN, + REF_START_TOKEN, AUDIO_START_TOKEN, + AUDIO_END_TOKEN, AUDIO_CONTEXT_TOKEN) + + + +def load_audio(audio_file, audio_processor): + audio_values, _ = librosa.load(audio_file, sr=16000) # sample rate should be 16000 + + audio_process_values = audio_processor(audio_values, sampling_rate=16000, return_tensors="pt") + input_features = audio_process_values['input_features'] + audio_len_after_cnn = audio_process_values['audio_len_after_cnn'] + audio_token_num = audio_process_values['audio_token_num'] + + + audio_input = {'audio_values': input_features, + 'audio_len_after_cnn': audio_len_after_cnn, + 'audio_token_num': audio_token_num, + } + return audio_input + + +class InternVLChatAudioModel(InternVLChatModel): + + def __init__(self, config: InternVLChatAudioConfig, vision_model=None, language_model=None, audio_model=None): + super().__init__(config, vision_model, language_model) + if audio_model is not None: + self.audio_model = audio_model + else: + self.audio_model = AudioWhisperModel(config.audio_config) + + audio_hidden_size = config.audio_config.d_model + llm_hidden_size = config.llm_config.hidden_size + self.mlp2 = nn.Sequential( + nn.LayerNorm(audio_hidden_size), + nn.Linear(audio_hidden_size, llm_hidden_size), + nn.GELU(), + nn.Linear(llm_hidden_size, llm_hidden_size) + ) # mlp2: audio feature mapping + + self.audio_context_token_id = None + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=0.02) + if hasattr(module, "bias") and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + elif isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + def extract_audio_feature(self, audio_values, audio_len_after_cnn): + + audio_values = audio_values.squeeze(1) + + #TODO: construct audio padding_mask in loader + max_len_in_batch = int(torch.max(audio_len_after_cnn).item()) + + padding_mask = torch.ones([audio_values.size(0), max_len_in_batch]).to(dtype=audio_values.dtype, + device=audio_values.device) + for index in range(len(audio_values)): + padding_mask[index, :int(audio_len_after_cnn[index].item())] = 0 + + last_hidden_state = self.audio_model(audio_values, padding_mask, audio_len_after_cnn) # (bs, max_token_num, 1280) + + audio_embeds = self.mlp2(last_hidden_state) + + return audio_embeds + + + def forward( + self, + pixel_values: torch.FloatTensor, + input_ids: torch.LongTensor = None, + audio_values: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + image_flags: Optional[torch.LongTensor] = None, + audio_flags: Optional[torch.LongTensor] = None, + audio_len_after_cnn: Optional[torch.LongTensor] = None, + audio_token_num: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + statistics: Optional[torch.LongTensor] = None, + loss_weight: Optional[List] = None, + loss_reduction_all_gather: Optional[bool] = False, + ) -> Union[Tuple, CausalLMOutputWithPast]: + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + image_flags = image_flags.squeeze(-1) + input_embeds = self.language_model.get_input_embeddings()(input_ids).clone() + + vit_embeds = self.extract_feature(pixel_values) + vit_embeds = vit_embeds[image_flags == 1] + vit_batch_size = pixel_values.shape[0] + + B, N, C = input_embeds.shape + input_embeds = input_embeds.reshape(B * N, C) + + if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0: + print(f'dynamic ViT batch size: {vit_batch_size}, images per sample: {vit_batch_size / B}, dynamic token length: {N}') + if statistics is not None: + num_samples, num_padding_tokens, num_padding_images = statistics.tolist() + self.num_samples += num_samples + print(f'total_samples={self.num_samples}, {num_samples=}, {num_padding_tokens=}, {num_padding_images=}') + + + input_ids = input_ids.reshape(B * N) + img_selected = (input_ids == self.img_context_token_id) + try: + input_embeds[img_selected] = input_embeds[img_selected] * 0.0 + vit_embeds.reshape(-1, C) + ignore_flag = False + except Exception as e: + vit_embeds = vit_embeds.reshape(-1, C) + print(f'warning: {e}, input_embeds[img_selected].shape={input_embeds[img_selected].shape}, ' + f'vit_embeds.shape={vit_embeds.shape}') + n_token = img_selected.sum() + input_embeds[img_selected] = input_embeds[img_selected] * 0.0 + vit_embeds[:n_token] + ignore_flag = True + + + + if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0: + audio_batch_size = audio_values.shape[0] + print(f'audio batch size: {audio_batch_size}, audios per sample: {audio_batch_size / B}') + + audio_embeds = self.extract_audio_feature(audio_values, audio_len_after_cnn) # (audio_num, n_frame, C) + + output_audios = [] + for i in range(len(audio_token_num)): + if audio_flags[i] > 0: + token_num = int(audio_token_num[i].item()) + audio = audio_embeds[i][:token_num] # 提取有效的token + output_audios.append(audio) + + if len(output_audios): + output_audios = torch.cat(output_audios, dim=0) + audio_selected = (input_ids == self.audio_context_token_id) + input_embeds[audio_selected] = input_embeds[audio_selected] * 0.0 + output_audios.reshape(-1, C) + + # # optional 2 + # valid_audio_mask = torch.zeros(audio_embeds.shape[:2], dtype=torch.bool) + # for i in range(len(audio_token_num)): + # token_num = int(audio_token_num[i].item()) + # valid_audio_mask[i, :token_num] = True + # output_audios_2 = audio_embeds[valid_audio_mask] + + + input_embeds = input_embeds.reshape(B, N, C) + + outputs = self.language_model( + inputs_embeds=input_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + logits = outputs.logits + + loss = None + if labels is not None and loss_weight is not None: + loss_weight = torch.tensor(loss_weight, + dtype=torch.float32, + device=labels.device) + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + shift_weights = loss_weight[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss(reduction='none') + shift_logits = shift_logits.view( + -1, self.language_model.config.vocab_size) + shift_labels = shift_labels.view(-1) + shift_weights = shift_weights.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + shift_weights = shift_weights.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + shift_weights_sum = shift_weights.sum() + if loss_reduction_all_gather: + dist.all_reduce(shift_weights_sum, op=dist.ReduceOp.AVG) + + loss = loss * shift_weights + loss = loss.sum() / shift_weights_sum + if ignore_flag: + loss = loss * 0.0 + elif labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.language_model.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + if ignore_flag: + loss = loss * 0.0 + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + + + @torch.no_grad() + def generate( + self, + pixel_values: torch.FloatTensor, + input_ids: torch.FloatTensor, + attention_mask: torch.LongTensor, + visual_features: Optional[torch.FloatTensor] = None, + audio_values: Optional[torch.FloatTensor] = None, + audio_len_after_cnn: Optional[bool] = None, + audio_token_num: Optional[bool] = None, + generation_config: Optional[GenerationConfig] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **generate_kwargs, + ) -> torch.LongTensor: + + assert self.img_context_token_id is not None + assert self.audio_context_token_id is not None + + vit_embeds = None + if visual_features is not None: + vit_embeds = visual_features + elif pixel_values is not None: + vit_embeds = self.extract_feature(pixel_values) + + input_embeds = self.language_model.get_input_embeddings()(input_ids) + B, N, C = input_embeds.shape + input_embeds = input_embeds.reshape(B * N, C) + + input_ids = input_ids.reshape(B * N) + + if vit_embeds is not None: + selected = (input_ids == self.img_context_token_id) + input_embeds[selected] = vit_embeds.reshape(-1, C) + + if audio_values is not None and audio_len_after_cnn is not None and audio_token_num is not None: + audio_embeds = self.extract_audio_feature(audio_values, audio_len_after_cnn) + output_audios = [] + for i in range(len(audio_token_num)): + token_num = int(audio_token_num[i].item()) + audio = audio_embeds[i][:token_num] + output_audios.append(audio) + output_audios = torch.cat(output_audios, dim=0) + selected = (input_ids == self.audio_context_token_id) + input_embeds[selected] = output_audios.reshape(-1, C) + + input_embeds = input_embeds.reshape(B, N, C) + + outputs = self.language_model.generate( + inputs_embeds=input_embeds, + attention_mask=attention_mask, + generation_config=generation_config, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + use_cache=True, + **generate_kwargs, + ) + + return outputs + + +class Chat: + def __init__( + self, + model, + tokenizer, + audio_processor, + device, + temperature=0.7, + top_p=0.5, + top_k=20, + repetition_penalty=1.1, + max_new_tokens=1024, + conv_style="internlm2-chat", + do_sample=True, + image_size=448, + dynamic_image_size=False, + scale_threshold='old', + min_dynamic_patch=1, + max_dynamic_patch=12, + use_thumbnail=True + ): + self.conv_style = conv_style + self.model = model + self.tokenizer = tokenizer + self.audio_processor = audio_processor + + self.device = device + self.dtype = model.dtype + + self.eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids(["<|im_end|>"])[0]] + + self.conv = get_conv_template(conv_style) + self.image_size = model.config.force_image_size or model.config.vision_config.image_size + self.dynamic_image_size = dynamic_image_size + self.scale_threshold = scale_threshold + self.min_dynamic_patch = min_dynamic_patch + self.max_dynamic_patch = max_dynamic_patch + self.use_thumbnail = model.config.use_thumbnail + + self.max_new_tokens = max_new_tokens + self.do_sample = do_sample + self.streamer = None + self.temperature = temperature + self.top_p = top_p + self.top_k = top_k + self.repetition_penalty = repetition_penalty + + def load_image(self, image_file): + if 's3://' in image_file: + image = Image.open(io.BytesIO(client.get(image_file))) + else: + if image_file.startswith('http') or image_file.startswith('https'): + response = requests.get(image_file) + image = Image.open(BytesIO(response.content)).convert('RGB') + else: + image = Image.open(image_file).convert('RGB') + + transform = build_transform(is_train=False, input_size=self.image_size) + + if self.dynamic_image_size: + if self.scale_threshold == "old": + images = dynamic_preprocess(image, min_num=self.min_dynamic_patch, max_num=self.max_dynamic_patch, + image_size=self.image_size, use_thumbnail=self.use_thumbnail) + else: + images = dynamic_preprocess(image, data_item=None, min_num=self.min_dynamic_patch, max_num=self.max_dynamic_patch, + image_size=self.image_size, use_thumbnail=self.use_thumbnail, + normalize_type=self.normalize_type, scale_threshold=float(self.scale_threshold)) #not detection data + # raise NotImplementedError + else: + images = [image] + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + num_patches = pixel_values.size(0) + return pixel_values + + def ask(self, text, conv, modal_type="audio"): + assert modal_type in ["text", "image", "audio", "image_audio"] + conversations = [] + + if len(conv.messages) > 0 or modal_type == "text": + conv.append_message(conv.roles[0], text) + elif modal_type == "image": + conv.append_message(conv.roles[0], "" + "\n" + text) + elif modal_type == "audio": + if text is None: + conv.append_message(conv.roles[0], "