diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..336bd9c --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,23 @@ +name: lint + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install pre-commit hook + run: | + pip install pre-commit + pre-commit install + - name: Linting + run: pre-commit run --all-files diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a20eec9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,166 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Images +images/ + +scripts/*ttf + +lvlm_zoo/ +LMUData/ +work_dirs + +batchscript-* +phoenix-slurm-* +LMUData diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..42b6815 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,28 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Current File", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal" + }, + { + "name": "main", + "type": "python", + "request": "launch", + "program": "run.py", + "console": "integratedTerminal", + "justMyCode": true, + "args": [ + "--data", "MMT-Bench_ALL", + "--model", "llava_v1.5_7b", + "--work-dir", "work_dirs/mmtbench" + ] + }, + ] +} \ No newline at end of file diff --git a/Quickstart.md b/Quickstart.md new file mode 100644 index 0000000..aa58acd --- /dev/null +++ b/Quickstart.md @@ -0,0 +1,118 @@ +# Quickstart + +## Prepare the Dataset + +You can download the MMT-Bench dataset in the following links: [HuggingFace](https://huggingface.co/datasets/Kaining/MMT-Bench/blob/main/MMT-Bench_VAL.tsv). **Note**: We only provide `VAL` split now. And we will support `TEST` split in the furture. + +Put the data under the `LMUData/` + +## Step 0. Installation & Setup essential keys + +**Installation.** + +```bash +pip install -e . +``` + +**Setup Keys.** + +To infer with API models (GPT-4v, Gemini-Pro-V, etc.) or use LLM APIs as the **judge or choice extractor**, you need to first setup API keys. VLMEvalKit will use an judge **LLM** to extract answer from the output if you set the key, otherwise it uses the **exact matching** mode (find "Yes", "No", "A", "B", "C"... in the output strings). **The exact matching can only be applied to the Yes-or-No tasks and the Multi-choice tasks.** +- You can place the required keys in `$VLMEvalKit/.env` or directly set them as the environment variable. If you choose to create a `.env` file, its content will look like: + + ```bash + # The .env file, place it under $VLMEvalKit + # Alles-apin-token, for intra-org use only + ALLES= + # API Keys of Proprietary VLMs + DASHSCOPE_API_KEY= + GOOGLE_API_KEY= + OPENAI_API_KEY= + OPENAI_API_BASE= + STEPAI_API_KEY= + ``` + +- Fill the blanks with your API keys (if necessary). Those API keys will be automatically loaded when doing the inference and evaluation. +## Step 1. Configuration + +**VLM Configuration**: All VLMs are configured in `vlmeval/config.py`, for some VLMs, you need to configure the code root (MiniGPT-4, PandaGPT, etc.) or the model_weight root (LLaVA-v1-7B, etc.) before conducting the evaluation. During evaluation, you should use the model name specified in `supported_VLM` in `vlmeval/config.py` to select the VLM. For MiniGPT-4 and InstructBLIP, you also need to modify the config files in `vlmeval/vlm/misc` to configure LLM path and ckpt path. + +Following VLMs require the configuration step: + +**Code Preparation & Installation**: InstructBLIP ([LAVIS](https://github.com/salesforce/LAVIS)), LLaVA ([LLaVA](https://github.com/haotian-liu/LLaVA)), MiniGPT-4 ([MiniGPT-4](https://github.com/Vision-CAIR/MiniGPT-4)), mPLUG-Owl2 ([mPLUG-Owl2](https://github.com/X-PLUG/mPLUG-Owl/tree/main/mPLUG-Owl2)), OpenFlamingo-v2 ([OpenFlamingo](https://github.com/mlfoundations/open_flamingo)), PandaGPT-13B ([PandaGPT](https://github.com/yxuansu/PandaGPT)), TransCore-M ([TransCore-M](https://github.com/PCIResearch/TransCore-M)). + +**Manual Weight Preparation & Configuration**: InstructBLIP, LLaVA-v1-7B, MiniGPT-4, PandaGPT-13B + +## Step 2. Evaluation + +We use `run.py` for evaluation. To use the script, you can use `$VLMEvalKit/run.py` or create a soft-link of the script (to use the script anywhere): + +**Arguments** + +- `--data (list[str])`: Set the dataset names that are supported in VLMEvalKit (defined in `vlmeval/utils/dataset_config.py`). +- `--model (list[str])`: Set the VLM names that are supported in VLMEvalKit (defined in `supported_VLM` in `vlmeval/config.py`). +- `--mode (str, default to 'all', choices are ['all', 'infer'])`: When `mode` set to "all", will perform both inference and evaluation; when set to "infer", will only perform the inference. +- `--nproc (int, default to 4)`: The number of threads for OpenAI API calling. + +**Command** + +You can run the script with `python` or `torchrun`: + +```bash +# When running with `python`, only one VLM instance is instantiated, and it might use multiple GPUs (depending on its default behavior). +# That is recommended for evaluating very large VLMs (like IDEFICS-80B-Instruct). + +# IDEFICS-80B-Instruct on MMT-Bench, Inference and Evalution +python run.py --data MMT-Bench_VAL --model idefics_80b_instruct --verbose +# IDEFICS-80B-Instruct on MMT-Bench, Inference only +python run.py --data MMT-Bench_VAL --model idefics_80b_instruct --verbose --mode infer + + +# IDEFICS-9B-Instruct, Qwen-VL-Chat, mPLUG-Owl2 on MMT-Bench. On a node with 8 GPU. Inference and Evaluation. +torchrun --nproc-per-node=8 run.py --data MMT-Bench_VAL --model idefics_80b_instruct qwen_chat mPLUG-Owl2 --verbose +# Qwen-VL-Chat on MMT-Bench. On a node with 2 GPU. Inference and Evaluation. +torchrun --nproc-per-node=2 run.py --data MMT-Bench_VAL --model qwen_chat --verbose +``` + +The evaluation results will be printed as logs, besides. **Result Files** will also be generated in the directory `$YOUR_WORKING_DIRECTORY/{model_name}`. Files ending with `.csv` contain the evaluated metrics. + +## Deploy a local language model as the judge / choice extractor +The default setting mentioned above uses OpenAI's GPT as the judge LLM. However, you can also deploy a local judge LLM with [LMDeploy](https://github.com/InternLM/lmdeploy). + +First install: +``` +pip install lmdeploy openai +``` + +And then deploy a local judge LLM with the single line of code. LMDeploy will automatically download the model from Huggingface. Assuming we use internlm2-chat-1_8b as the judge, port 23333, and the key sk-123456 (the key must start with "sk-" and follow with any number you like): +``` +lmdeploy serve api_server internlm/internlm2-chat-1_8b --server-port 23333 +``` + +You need to get the model name registered by LMDeploy with the following python code: +``` +from openai import OpenAI +client = OpenAI( + api_key='sk-123456', + base_url="http://0.0.0.0:23333/v1" +) +model_name = client.models.list().data[0].id +``` + +Now set some environment variables to tell VLMEvalKit how to use the local judge LLM. As mentioned above, you can also set them in `$VLMEvalKit/.env` file: +``` +OPENAI_API_KEY=sk-123456 +OPENAI_API_BASE=http://0.0.0.0:23333/v1/chat/completions +LOCAL_LLM= +``` + +Finally, you can run the commands in step 2 to evaluate your VLM with the local judge LLM. + +Note that + +- If you hope to deploy the judge LLM in a single GPU and evaluate your VLM on other GPUs because of limited GPU memory, try `CUDA_VISIBLE_DEVICES=x` like +``` +CUDA_VISIBLE_DEVICES=0 lmdeploy serve api_server internlm/internlm2-chat-1_8b --server-port 23333 +CUDA_VISIBLE_DEVICES=1,2,3 torchrun --nproc-per-node=3 run.py --data HallusionBench --model qwen_chat --verbose +``` +- If the local judge LLM is not good enough in following the instructions, the evaluation may fail. Please report such failures (e.g., by issues). +- It's possible to deploy the judge LLM in different ways, e.g., use a private LLM (not from HuggingFace) or use a quantized LLM. Please refer to the [LMDeploy doc](https://lmdeploy.readthedocs.io/en/latest/serving/api_server.html). You can use any other deployment framework if they support OpenAI API. diff --git a/README.md b/README.md index 706a1a9..3286462 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,28 @@ # MMT-Bench -This repository is the official implementation of [MMT-Bench](xxx). +

+ Quick Start | + HomePage | + arXiv | + Dataset | + Citation
-> [MMT-Bench: A Multimodal MultiTask Benchmark for Comprehensive Evaluation of Large Vision-Language Models](xxx) +

+ + +This repository is the official implementation of [MMT-Bench](https://arxiv.org/abs/2404.16006). + +> [MMT-Bench: A Multimodal MultiTask Benchmark for Comprehensive Evaluation of Large Vision-Language Models](https://arxiv.org/abs/2404.16006) > Kaining Ying\*, Fanqing Meng\*, Jin Wang\*, Zhiqian Li, Han Lin, Yue Yang, Hao Zhang, Wenbo Zhang, Yuqi Lin, Shuo Liu, jiayi lei, Quanfeng Lu, Peng Gao, Runjian Chen, Peng Xu, Renrui Zhang, Haozhe Zhang, Yali Wang, Yu Qiao, Ping Luo, Kaipeng Zhang\#, Wenqi Shao\# -> \* KY, FM and JW contribute equally. \# KZ (zhangkaipeng@pjlab.org.cn) and WS (shaowenqi@pjlab.org.cn) are correponding authors. +> \* KY, FM and JW contribute equally. +> \# WS (shaowenqi@pjlab.org.cn) and KZ (zhangkaipeng@pjlab.org.cn) are correponding authors. ![overview](assets/overview.jpg) ## 🏆 Leaderboard +### Full Set + | Rank | Model | Overall | |------|-----------------------|---------| | 1 | InternVL-Chat-v1.2 | 63.4 | @@ -45,10 +58,21 @@ This repository is the official implementation of [MMT-Bench](xxx). | 31 | Frequency Guess | 31.7 | | 32 | Random Guess | 28.5 | +### VAL Split + +Coming soon. + +### TEST Split + +Coming soon. + ## 💡 News +- `2024/04/24`: The technical report of [MMT-Bench](https://arxiv.org/abs/2404.16006) is released! And check our [project page](https://mmt-bench.github.io/)! +- `2024/04/26`: We release the evaluation code and the `VAL` split. + ## 🚀 Quick Start @@ -59,11 +83,18 @@ Please refer to [this](Quickstart.md) to quick start. ## 💐 Acknowledgement We expressed sincerely gratitude for the projects listed following: -- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) provides useful out-of-box tools and implements many adavanced LVLMs. +- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) provides useful out-of-box tools and implements many adavanced LVLMs. Thanks for their selfless dedication. ## 🖊️ Citation If you feel MMT-Bench useful in your project or research, please kindly use the following BibTeX entry to cite our paper. Thanks! ``` -xxx +@misc{mmtbench, + title={MMT-Bench: A Comprehensive Multimodal Benchmark for Evaluating Large Vision-Language Models Towards Multitask AGI}, + author={Kaining Ying and Fanqing Meng and Jin Wang and Zhiqian Li and Han Lin and Yue Yang and Hao Zhang and Wenbo Zhang and Yuqi Lin and Shuo Liu and Jiayi Lei and Quanfeng Lu and Runjian Chen and Peng Xu and Renrui Zhang and Haozhe Zhang and Peng Gao and Yali Wang and Yu Qiao and Ping Luo and Kaipeng Zhang and Wenqi Shao}, + year={2024}, + eprint={2404.16006}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} ``` \ No newline at end of file diff --git a/assets/LOGO.svg b/assets/LOGO.svg new file mode 100644 index 0000000..39b6226 --- /dev/null +++ b/assets/LOGO.svg @@ -0,0 +1,24 @@ + + + +Created with Fabric.js 5.3.0 + + + + + + + + + + + + + VLMEvalKit + \ No newline at end of file diff --git a/assets/apple.jpg b/assets/apple.jpg new file mode 100644 index 0000000..34e1859 Binary files /dev/null and b/assets/apple.jpg differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..63e3886 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +einops +gradio==4.15.0 +huggingface_hub +matplotlib +numpy>=1.23.4 +omegaconf +openai==1.3.5 +opencv-python>=4.4.0.46 +openpyxl +pandas>=1.5.3 +pillow +portalocker +protobuf +pycocoevalcap +python-dotenv +requests +rich +seaborn +sentencepiece +sty +tabulate +tiktoken +timeout-decorator +torch>=2.0.1 +tqdm +transformers +typing_extensions==4.7.1 +validators +visual_genome +xlsxwriter +xtuner diff --git a/run.py b/run.py new file mode 100644 index 0000000..6385f7d --- /dev/null +++ b/run.py @@ -0,0 +1,147 @@ +import os +os.environ['LMUData'] = "LMUData/" + +import torch +import torch.distributed as dist +from vlmeval.smp import * +from vlmeval.evaluate import * +from vlmeval.inference import infer_data_job +from vlmeval.config import supported_VLM +from vlmeval.utils import dataset_URLs, DATASET_TYPE, abbr2full, MMMU_result_transfer + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--data', type=str, nargs='+', required=True) + parser.add_argument('--model', type=str, nargs='+', required=True) + parser.add_argument('--work-dir', type=str, default='.', help='select the output directory') + parser.add_argument('--mode', type=str, default='all', choices=['all', 'infer']) + parser.add_argument('--nproc', type=int, default=4, help='Parallel API calling') + parser.add_argument('--retry', type=int, default=None, help='retry numbers for API VLMs') + parser.add_argument('--judge', type=str, default=None) + parser.add_argument('--ignore', action='store_true', help='Ignore failed indices. ') + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +def main(): + logger = get_logger('RUN') + + args = parse_args() + assert len(args.data), '--data should be a list of data files' + + if args.retry is not None: + for k, v in supported_VLM.items(): + if hasattr(v, 'keywords') and 'retry' in v.keywords: + v.keywords['retry'] = args.retry + supported_VLM[k] = v + if hasattr(v, 'keywords') and 'verbose' in v.keywords: + v.keywords['verbose'] = args.verbose + supported_VLM[k] = v + + rank, world_size = get_rank_and_world_size() + if world_size > 1: + local_rank = os.environ.get('LOCAL_RANK', 0) + torch.cuda.set_device(local_rank) + dist.init_process_group(backend='nccl', timeout=datetime.timedelta(seconds=10800)) + + for _, model_name in enumerate(args.model): + model = None + + pred_root = osp.join(args.work_dir, model_name) + os.makedirs(pred_root, exist_ok=True) + + for _, dataset_name in enumerate(args.data): + custom_flag = False + + if dataset_name not in dataset_URLs: + dataset_name = abbr2full(dataset_name) + + if dataset_name not in dataset_URLs: + logger.warning(f'Dataset {dataset_name} is not officially supported. ') + file_path = osp.join(LMUDataRoot(), f'{dataset_name}.tsv') + if not osp.exists(file_path): + logger.error(f'Cannot find the local dataset {dataset_name}. ') + continue + else: + custom_flag = True + + result_file = f'{pred_root}/{model_name}_{dataset_name}.xlsx' + + if model is None: + model = model_name # which is only a name + + model = infer_data_job( + model, + work_dir=pred_root, + model_name=model_name, + dataset_name=dataset_name, + verbose=args.verbose, + api_nproc=args.nproc, + ignore_failed=args.ignore) + + if rank == 0: + if dataset_name in ['MMMU_TEST']: + result_json = MMMU_result_transfer(result_file) + logger.info(f'Transfer MMMU_TEST result to json for official evaluation, json file saved in {result_json}') # noqa: E501 + + if dataset_name in ['MMBench_TEST_CN', 'MMBench_TEST_EN', 'MMMU_TEST']: + if not MMBenchOfficialServer(): + logger.error( + f'Can not evaluate {dataset_name} on non-official servers, ' + 'will skip the evaluation. ' + ) + continue + + judge_kwargs = { + 'nproc': args.nproc, + 'verbose': args.verbose, + } + if args.retry is not None: + judge_kwargs['retry'] = args.retry + if args.judge is not None: + judge_kwargs['model'] = args.judge + else: + if DATASET_TYPE(dataset_name) in ['multi-choice', 'Y/N']: + judge_kwargs['model'] = 'chatgpt-0613' + elif listinstr(['MMVet', 'MathVista', 'LLaVABench'], dataset_name): + judge_kwargs['model'] = 'gpt-4-turbo' + if 'OPENAI_API_KEY_JUDGE' in os.environ and len(os.environ['OPENAI_API_KEY_JUDGE']): + judge_kwargs['key'] = os.environ['OPENAI_API_KEY_JUDGE'] + if 'OPENAI_API_BASE_JUDGE' in os.environ and len(os.environ['OPENAI_API_BASE_JUDGE']): + judge_kwargs['api_base'] = os.environ['OPENAI_API_BASE_JUDGE'] + + if rank == 0 and args.mode == 'all': + if DATASET_TYPE(dataset_name) == 'multi-choice': + dataset_name = 'default' if custom_flag else dataset_name + multiple_choice_eval( + result_file, + dataset=dataset_name, + **judge_kwargs) + + elif DATASET_TYPE(dataset_name) == 'Y/N': + YOrN_eval( + result_file, + dataset=dataset_name, + **judge_kwargs) + + elif DATASET_TYPE(dataset_name) == 'Caption': + COCO_eval(result_file) + elif dataset_name == 'MMVet': + MMVet_eval(result_file, **judge_kwargs) + elif dataset_name == 'OCRBench': + OCRBench_eval(result_file) + elif listinstr(['OCRVQA', 'TextVQA', 'ChartQA', 'DocVQA', 'InfoVQA'], dataset_name): + VQAEval(result_file, dataset_name) + elif listinstr(['MathVista'], dataset_name): + MathVista_eval(result_file, **judge_kwargs) + elif listinstr(['LLaVABench'], dataset_name): + LLaVABench_eval(result_file, **judge_kwargs) + else: + logger.error(f'Dataset {dataset_name} is not handled by evaluator, will be skipped. ') + + +if __name__ == '__main__': + load_env() + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..72b742e --- /dev/null +++ b/setup.py @@ -0,0 +1,122 @@ +import re +import sys +from os.path import exists +from setuptools import find_packages, setup + + +def parse_requirements(fname='requirements.txt', with_version=True): + """Parse the package dependencies listed in a requirements file but strips + specific versioning information. + + Args: + fname (str): path to requirements file + with_version (bool, default=False): if True include version specs + + Returns: + List[str]: list of requirements items + + CommandLine: + python -c "import setup; print(setup.parse_requirements())" + """ + + require_fpath = fname + + def parse_line(line): + """Parse information from a line in a requirements text file.""" + if line.startswith('-r '): + # Allow specifying requirements in other files + target = line.split(' ')[1] + for info in parse_require_file(target): + yield info + else: + info = {'line': line} + if line.startswith('-e '): + info['package'] = line.split('#egg=')[1] + elif '@git+' in line: + info['package'] = line + else: + # Remove versioning from the package + pat = '(' + '|'.join(['>=', '==', '>']) + ')' + parts = re.split(pat, line, maxsplit=1) + parts = [p.strip() for p in parts] + + info['package'] = parts[0] + if len(parts) > 1: + op, rest = parts[1:] + if ';' in rest: + # Handle platform specific dependencies + # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies + version, platform_deps = map(str.strip, + rest.split(';')) + info['platform_deps'] = platform_deps + else: + version = rest # NOQA + info['version'] = (op, version) + yield info + + def parse_require_file(fpath): + with open(fpath, 'r') as f: + for line in f.readlines(): + line = line.strip() + if line and not line.startswith('#'): + for info in parse_line(line): + yield info + + def gen_packages_items(): + if exists(require_fpath): + for info in parse_require_file(require_fpath): + parts = [info['package']] + if with_version and 'version' in info: + parts.extend(info['version']) + if not sys.version.startswith('3.4'): + # apparently package_deps are broken in 3.4 + platform_deps = info.get('platform_deps') + if platform_deps is not None: + parts.append(';' + platform_deps) + item = ''.join(parts) + yield item + + packages = list(gen_packages_items()) + return packages + + +with open('README.md') as f: + readme = f.read() + + +def do_setup(): + setup( + name='vlmeval', + version='0.1.0', + description='OpenCompass VLM Evaluation Kit', + author='Haodong Duan', + author_email='dhd.efz@gmail.com', + maintainer='Haodong Duan', + maintainer_email='dhd.efz@gmail.com', + long_description=readme, + long_description_content_type='text/markdown', + cmdclass={}, + install_requires=parse_requirements('requirements.txt'), + setup_requires=[], + python_requires='>=3.7.0', + packages=find_packages(exclude=[ + 'test*', + 'paper_test*', + ]), + keywords=['AI', 'NLP', 'in-context learning'], + entry_points={ + 'console_scripts': [] + }, + classifiers=[ + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + ]) + + +if __name__ == '__main__': + do_setup() diff --git a/vlmeval/__init__.py b/vlmeval/__init__.py new file mode 100644 index 0000000..336a61a --- /dev/null +++ b/vlmeval/__init__.py @@ -0,0 +1,13 @@ +try: + import torch +except ImportError: + pass + +from .smp import * +from .api import * +from .evaluate import * +from .utils import * +from .vlm import * +from .config import * + +load_env() diff --git a/vlmeval/api/__init__.py b/vlmeval/api/__init__.py new file mode 100644 index 0000000..63a2e31 --- /dev/null +++ b/vlmeval/api/__init__.py @@ -0,0 +1,14 @@ +from .gpt import OpenAIWrapper, GPT4V +from .gpt_int import OpenAIWrapperInternal, GPT4V_Internal +from .hf_chat_model import HFChatModel +from .gemini import GeminiWrapper, GeminiProVision +from .qwen_vl_api import QwenVLWrapper, QwenVLAPI +from .qwen_api import QwenAPI +from .stepai import Step1V +from .claude import Claude_Wrapper, Claude3V + +__all__ = [ + 'OpenAIWrapper', 'HFChatModel', 'OpenAIWrapperInternal', 'GeminiWrapper', + 'GPT4V', 'GPT4V_Internal', 'GeminiProVision', 'QwenVLWrapper', 'QwenVLAPI', + 'QwenAPI', 'Step1V', 'Claude3V', 'Claude_Wrapper' +] diff --git a/vlmeval/api/base.py b/vlmeval/api/base.py new file mode 100644 index 0000000..b7933bf --- /dev/null +++ b/vlmeval/api/base.py @@ -0,0 +1,176 @@ +import time +import random as rd +from abc import abstractmethod +import os.path as osp +import copy as cp +from ..smp import get_logger, parse_file + + +class BaseAPI: + + allowed_types = ['text', 'image'] + INTERLEAVE = True + INSTALL_REQ = False + + def __init__(self, + retry=10, + wait=3, + system_prompt=None, + verbose=True, + fail_msg='Failed to obtain answer via API.', + **kwargs): + """Base Class for all APIs. + + Args: + retry (int, optional): The retry times for `generate_inner`. Defaults to 10. + wait (int, optional): The wait time after each failed retry of `generate_inner`. Defaults to 3. + system_prompt (str, optional): Defaults to None. + verbose (bool, optional): Defaults to True. + fail_msg (str, optional): The message to return when failed to obtain answer. + Defaults to 'Failed to obtain answer via API.'. + **kwargs: Other kwargs for `generate_inner`. + """ + + self.wait = wait + self.retry = retry + self.system_prompt = system_prompt + self.verbose = verbose + self.fail_msg = fail_msg + self.logger = get_logger('ChatAPI') + + if len(kwargs): + self.logger.info(f'BaseAPI received the following kwargs: {kwargs}') + self.logger.info('Will try to use them as kwargs for `generate`. ') + self.default_kwargs = kwargs + + @abstractmethod + def generate_inner(self, inputs, **kwargs): + """The inner function to generate the answer. + + Returns: + tuple(int, str, str): ret_code, response, log + """ + self.logger.warning('For APIBase, generate_inner is an abstract method. ') + assert 0, 'generate_inner not defined' + ret_code, answer, log = None, None, None + # if ret_code is 0, means succeed + return ret_code, answer, log + + def working(self): + """If the API model is working, return True, else return False. + + Returns: + bool: If the API model is working, return True, else return False. + """ + retry = 3 + while retry > 0: + ret = self.generate('hello') + if ret is not None and ret != '' and self.fail_msg not in ret: + return True + retry -= 1 + return False + + def check_content(self, msgs): + """Check the content type of the input. Four types are allowed: str, dict, liststr, listdict. + + Args: + msgs: Raw input messages. + + Returns: + str: The message type. + """ + if isinstance(msgs, str): + return 'str' + if isinstance(msgs, dict): + return 'dict' + if isinstance(msgs, list): + types = [self.check_content(m) for m in msgs] + if all(t == 'str' for t in types): + return 'liststr' + if all(t == 'dict' for t in types): + return 'listdict' + return 'unknown' + + def preproc_content(self, inputs): + """Convert the raw input messages to a list of dicts. + + Args: + inputs: raw input messages. + + Returns: + list(dict): The preprocessed input messages. Will return None if failed to preprocess the input. + """ + if self.check_content(inputs) == 'str': + return [dict(type='text', value=inputs)] + elif self.check_content(inputs) == 'dict': + assert 'type' in inputs and 'value' in inputs + return [inputs] + elif self.check_content(inputs) == 'liststr': + res = [] + for s in inputs: + mime, pth = parse_file(s) + if mime is None or mime == 'unknown': + res.append(dict(type='text', value=s)) + else: + res.append(dict(type=mime.split('/')[0], value=pth)) + return res + elif self.check_content(inputs) == 'listdict': + for item in inputs: + assert 'type' in item and 'value' in item + mime, s = parse_file(item['value']) + if mime is None: + assert item['type'] == 'text', item['value'] + else: + assert mime.split('/')[0] == item['type'] + item['value'] = s + return inputs + else: + return None + + def generate(self, message, **kwargs1): + """The main function to generate the answer. Will call `generate_inner` with the preprocessed input messages. + + Args: + message: raw input messages. + + Returns: + str: The generated answer of the Failed Message if failed to obtain answer. + """ + assert self.check_content(message) in ['str', 'dict', 'liststr', 'listdict'], f'Invalid input type: {message}' + message = self.preproc_content(message) + assert message is not None and self.check_content(message) == 'listdict' + for item in message: + assert item['type'] in self.allowed_types, f'Invalid input type: {item["type"]}' + + # merge kwargs + kwargs = cp.deepcopy(self.default_kwargs) + kwargs.update(kwargs1) + + answer = None + # a very small random delay [0s - 0.5s] + T = rd.random() * 0.5 + time.sleep(T) + + for i in range(self.retry): + try: + ret_code, answer, log = self.generate_inner(message, **kwargs) + if ret_code == 0 and self.fail_msg not in answer and answer != '': + if self.verbose: + print(answer) + return answer + elif self.verbose: + if not isinstance(log, str): + try: + log = log.text + except: + self.logger.warning(f'Failed to parse {log} as an http response. ') + self.logger.info(f'RetCode: {ret_code}\nAnswer: {answer}\nLog: {log}') + except Exception as err: + if self.verbose: + self.logger.error(f'An error occured during try {i}:') + self.logger.error(err) + # delay before each retry + T = rd.random() * self.wait * 2 + time.sleep(T) + + return self.fail_msg if answer in ['', None] else answer diff --git a/vlmeval/api/claude.py b/vlmeval/api/claude.py new file mode 100644 index 0000000..f1e5892 --- /dev/null +++ b/vlmeval/api/claude.py @@ -0,0 +1,99 @@ +from vlmeval.smp import * +from vlmeval.api.base import BaseAPI +from time import sleep +import base64 +import mimetypes + +url = 'https://openxlab.org.cn/gw/alles-apin-hub/v1/claude/v1/text/chat' +headers = { + 'alles-apin-token': '', + 'Content-Type': 'application/json' +} + + +class Claude_Wrapper(BaseAPI): + + is_api: bool = True + + def __init__(self, + model: str = 'claude-3-opus-20240229', + key: str = None, + retry: int = 10, + wait: int = 3, + system_prompt: str = None, + verbose: bool = True, + temperature: float = 0, + max_tokens: int = 1024, + **kwargs): + + self.model = model + self.headers = headers + self.temperature = temperature + self.max_tokens = max_tokens + if key is not None: + self.key = key + else: + self.key = os.environ.get('ALLES', '') + self.headers['alles-apin-token'] = self.key + + super().__init__(retry=retry, wait=wait, verbose=verbose, system_prompt=system_prompt, **kwargs) + + def build_msgs(self, msgs_raw): + + messages = [] + message = {'role': 'user', 'content': []} + for msg in msgs_raw: + if msg['type'] == 'image': + pth = msg['value'] + suffix = osp.splitext(pth)[-1].lower() + media_type = mimetypes.types_map.get(suffix, None) + assert media_type is not None + + item = { + 'type': 'image', + 'source': {'type': 'base64', 'media_type': media_type, 'data': encode_image_file_to_base64(pth)} + } + + elif msg['type'] == 'text': + item = {'type': 'text', 'text': msg['value']} + else: + raise NotImplementedError(f'Unsupported message type: {msg["type"]}') + + message['content'].append(item) + messages.append(message) + return messages + + def generate_inner(self, inputs, **kwargs) -> str: + + payload = json.dumps({ + 'model': self.model, + 'max_tokens': self.max_tokens, + 'messages': self.build_msgs(msgs_raw=inputs), + **kwargs + }) + response = requests.request('POST', url, headers=headers, data=payload) + + ret_code = response.status_code + retry = self.retry + while ret_code == 429 and retry > 0: + sleep(15) + response = requests.request('POST', url, headers=headers, data=payload) + ret_code = response.status_code + retry -= 1 + + ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code + answer = self.fail_msg + + try: + resp_struct = json.loads(response.text) + answer = resp_struct['data']['content'][0]['text'].strip() + except: + pass + + return ret_code, answer, response + + +class Claude3V(Claude_Wrapper): + + def generate(self, message, dataset=None): + return super(Claude_Wrapper, self).generate(message) diff --git a/vlmeval/api/gemini.py b/vlmeval/api/gemini.py new file mode 100644 index 0000000..04aa9db --- /dev/null +++ b/vlmeval/api/gemini.py @@ -0,0 +1,65 @@ +from vlmeval.smp import * +from vlmeval.api.base import BaseAPI + +headers = 'Content-Type: application/json' + + +class GeminiWrapper(BaseAPI): + + is_api: bool = True + + def __init__(self, + retry: int = 5, + wait: int = 5, + key: str = None, + verbose: bool = True, + temperature: float = 0.0, + system_prompt: str = None, + max_tokens: int = 1024, + proxy: str = None, + **kwargs): + + self.fail_msg = 'Failed to obtain answer via API. ' + self.max_tokens = max_tokens + self.temperature = temperature + if key is None: + key = os.environ.get('GOOGLE_API_KEY', None) + assert key is not None + self.api_key = key + if proxy is not None: + proxy_set(proxy) + super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs) + + def build_msgs(self, inputs): + messages = [] if self.system_prompt is None else [self.system_prompt] + for inp in inputs: + if inp['type'] == 'text': + messages.append(inp['value']) + elif inp['type'] == 'image': + messages.append(Image.open(inp['value'])) + return messages + + def generate_inner(self, inputs, **kwargs) -> str: + import google.generativeai as genai + assert isinstance(inputs, list) + pure_text = np.all([x['type'] == 'text' for x in inputs]) + genai.configure(api_key=self.api_key) + model = genai.GenerativeModel('gemini-pro') if pure_text else genai.GenerativeModel('gemini-pro-vision') + messages = self.build_msgs(inputs) + gen_config = dict(max_output_tokens=self.max_tokens, temperature=self.temperature) + gen_config.update(kwargs) + try: + answer = model.generate_content(messages, generation_config=genai.types.GenerationConfig(**gen_config)).text + return 0, answer, 'Succeeded! ' + except Exception as err: + if self.verbose: + self.logger.error(err) + self.logger.error(f'The input messages are {inputs}.') + + return -1, '', '' + + +class GeminiProVision(GeminiWrapper): + + def generate(self, message, dataset=None): + return super(GeminiProVision, self).generate(message) diff --git a/vlmeval/api/gpt.py b/vlmeval/api/gpt.py new file mode 100644 index 0000000..eefeba1 --- /dev/null +++ b/vlmeval/api/gpt.py @@ -0,0 +1,175 @@ +from ..smp import * +import os +import sys +from .base import BaseAPI + +APIBASES = { + 'OFFICIAL': 'https://api.openai.com/v1/chat/completions', +} + + +def GPT_context_window(model): + length_map = { + 'gpt-4-1106-preview': 128000, + 'gpt-4-vision-preview': 128000, + 'gpt-4': 8192, + 'gpt-4-32k': 32768, + 'gpt-4-0613': 8192, + 'gpt-4-32k-0613': 32768, + 'gpt-3.5-turbo-1106': 16385, + 'gpt-3.5-turbo': 4096, + 'gpt-3.5-turbo-16k': 16385, + 'gpt-3.5-turbo-instruct': 4096, + 'gpt-3.5-turbo-0613': 4096, + 'gpt-3.5-turbo-16k-0613': 16385, + } + if model in length_map: + return length_map[model] + else: + return 128000 + + +class OpenAIWrapper(BaseAPI): + + is_api: bool = True + + def __init__(self, + model: str = 'gpt-3.5-turbo-0613', + retry: int = 5, + wait: int = 5, + key: str = None, + verbose: bool = True, + system_prompt: str = None, + temperature: float = 0, + timeout: int = 60, + api_base: str = None, + max_tokens: int = 1024, + img_size: int = 512, + img_detail: str = 'low', + **kwargs): + + self.model = model + self.cur_idx = 0 + self.fail_msg = 'Failed to obtain answer via API. ' + self.max_tokens = max_tokens + self.temperature = temperature + + env_key = os.environ.get('OPENAI_API_KEY', '') + openai_key = env_key if key is None else key + self.openai_key = openai_key + assert img_size > 0 or img_size == -1 + self.img_size = img_size + assert img_detail in ['high', 'low'] + self.img_detail = img_detail + + self.vision = False + if model == 'gpt-4-vision-preview': + self.vision = True + self.timeout = timeout + + assert isinstance(openai_key, str) and (openai_key.startswith('sk-') or openai_key.startswith('sb-')), ( + f'Illegal openai_key {openai_key}. ' + 'Please set the environment variable OPENAI_API_KEY to your openai key. ' + ) + super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs) + + if api_base is None: + if 'OPENAI_API_BASE' in os.environ and os.environ['OPENAI_API_BASE'] != '': + self.logger.error('Environment variable OPENAI_API_BASE is set. Will use it as api_base. ') + api_base = os.environ['OPENAI_API_BASE'] + else: + api_base = 'OFFICIAL' + + assert api_base is not None + + if api_base in APIBASES: + self.api_base = APIBASES[api_base] + elif api_base.startswith('http'): + self.api_base = api_base + else: + self.logger.error('Unknown API Base. ') + sys.exit(-1) + + # inputs can be a lvl-2 nested list: [content1, content2, content3, ...] + # content can be a string or a list of image & text + def prepare_inputs(self, inputs): + input_msgs = [] + if self.system_prompt is not None: + input_msgs.append(dict(role='system', content=self.system_prompt)) + has_images = np.sum([x['type'] == 'image' for x in inputs]) + if has_images: + content_list = [] + for msg in inputs: + if msg['type'] == 'text': + content_list.append(dict(type='text', text=msg['value'])) + elif msg['type'] == 'image': + from PIL import Image + img = Image.open(msg['value']) + b64 = encode_image_to_base64(img, target_size=self.img_size) + img_struct = dict(url=f'data:image/jpeg;base64,{b64}', detail=self.img_detail) + content_list.append(dict(type='image_url', image_url=img_struct)) + input_msgs.append(dict(role='user', content=content_list)) + else: + assert all([x['type'] == 'text' for x in inputs]) + text = '\n'.join([x['value'] for x in inputs]) + input_msgs.append(dict(role='user', content=text)) + return input_msgs + + def generate_inner(self, inputs, **kwargs) -> str: + input_msgs = self.prepare_inputs(inputs) + temperature = kwargs.pop('temperature', self.temperature) + max_tokens = kwargs.pop('max_tokens', self.max_tokens) + + context_window = GPT_context_window(self.model) + max_tokens = min(max_tokens, context_window - self.get_token_len(inputs)) + if 0 < max_tokens <= 100: + self.logger.warning( + 'Less than 100 tokens left, ' + 'may exceed the context window with some additional meta symbols. ' + ) + if max_tokens <= 0: + return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. ' + + headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.openai_key}'} + payload = dict( + model=self.model, + messages=input_msgs, + max_tokens=max_tokens, + n=1, + temperature=temperature, + **kwargs) + response = requests.post(self.api_base, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1) + ret_code = response.status_code + ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code + answer = self.fail_msg + try: + resp_struct = json.loads(response.text) + answer = resp_struct['choices'][0]['message']['content'].strip() + except: + pass + return ret_code, answer, response + + def get_token_len(self, inputs) -> int: + import tiktoken + try: + enc = tiktoken.encoding_for_model(self.model) + except: + enc = tiktoken.encoding_for_model('gpt-4') + assert isinstance(inputs, list) + tot = 0 + for item in inputs: + if item['type'] == 'text': + tot += len(enc.encode(item['value'])) + elif item['type'] == 'image': + tot += 85 + if self.img_detail == 'high': + img = Image.open(item['value']) + npatch = np.ceil(img.size[0] / 512) * np.ceil(img.size[1] / 512) + tot += npatch * 170 + return tot + + +class GPT4V(OpenAIWrapper): + + def generate(self, message, dataset=None): + return super(GPT4V, self).generate(message) diff --git a/vlmeval/api/gpt_int.py b/vlmeval/api/gpt_int.py new file mode 100644 index 0000000..f520747 --- /dev/null +++ b/vlmeval/api/gpt_int.py @@ -0,0 +1,94 @@ +import json +import warnings +import requests +from ..smp import * +from .gpt import GPT_context_window, OpenAIWrapper + +url = 'http://ecs.sv.us.alles-apin.openxlab.org.cn/v1/openai/v2/text/chat' +headers = { + 'Content-Type': 'application/json' +} + + +class OpenAIWrapperInternal(OpenAIWrapper): + + is_api: bool = True + + def __init__(self, + model: str = 'gpt-3.5-turbo-0613', + retry: int = 5, + wait: int = 3, + verbose: bool = True, + system_prompt: str = None, + temperature: float = 0, + timeout: int = 60, + max_tokens: int = 1024, + img_size: int = 512, + img_detail: str = 'low', + **kwargs): + + self.model = model + if 'KEYS' in os.environ and osp.exists(os.environ['KEYS']): + keys = load(os.environ['KEYS']) + headers['alles-apin-token'] = keys.get('alles-apin-token', '') + elif 'ALLES' in os.environ: + headers['alles-apin-token'] = os.environ['ALLES'] + self.headers = headers + self.temperature = temperature + self.timeout = timeout + self.max_tokens = max_tokens + + assert img_size > 0 or img_size == -1 + self.img_size = img_size + assert img_detail in ['high', 'low'] + self.img_detail = img_detail + + self.vision = False + if model == 'gpt-4-vision-preview': + self.vision = True + + super(OpenAIWrapper, self).__init__( + wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs) + + def generate_inner(self, inputs, **kwargs) -> str: + input_msgs = self.prepare_inputs(inputs) + + temperature = kwargs.pop('temperature', self.temperature) + max_tokens = kwargs.pop('max_tokens', self.max_tokens) + + # Held out 100 tokens as buffer + context_window = GPT_context_window(self.model) + max_tokens = min(max_tokens, context_window - self.get_token_len(inputs)) + if 0 < max_tokens <= 100: + print('Less than 100 tokens left, may exceed the context window with some additional meta symbols. ') + if max_tokens <= 0: + return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. ' + + payload = dict( + model=self.model, + messages=input_msgs, + max_tokens=max_tokens, + n=1, + stop=None, + timeout=self.timeout, + temperature=temperature, + **kwargs) + + response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1) + ret_code = response.status_code + ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code + + answer = self.fail_msg + try: + resp_struct = json.loads(response.text) + assert resp_struct['msg'] == 'ok' and resp_struct['msgCode'] == '10000', resp_struct + answer = resp_struct['data']['choices'][0]['message']['content'].strip() + except: + pass + return ret_code, answer, response + + +class GPT4V_Internal(OpenAIWrapperInternal): + + def generate(self, message, dataset=None): + return super(GPT4V_Internal, self).generate(message) diff --git a/vlmeval/api/hf_chat_model.py b/vlmeval/api/hf_chat_model.py new file mode 100644 index 0000000..9f2ae3c --- /dev/null +++ b/vlmeval/api/hf_chat_model.py @@ -0,0 +1,245 @@ +import os +import sys +import os.path as osp +import torch +from ..smp import * + + +def get_gpu_num(model_name): + model_name = model_name.lower() + kws = { + 8: ['65b', '70b'], + 4: ['30b', '33b', '35b', '40b'], + 2: ['13b', '14b', '20b'], + 1: ['6b', '7b', 'moss'], + } + for k in [8, 4, 2, 1]: + for keyword in kws[k]: + if keyword in model_name: + return k + return 8 + + +validated_llms = [ + 'internlm/internlm-chat-7b', 'internlm/internlm-chat-7b-8k', 'internlm/internlm-chat-20b', + 'Qwen/Qwen-7B-Chat', 'Qwen/Qwen-14B-Chat', + 'THUDM/chatglm2-6b', 'THUDM/chatglm2-6b-32k', 'THUDM/chatglm3-6b', 'THUDM/chatglm3-6b-32k', + 'baichuan-inc/Baichuan2-7B-Chat', 'baichuan-inc/Baichuan2-13B-Chat', + 'lmsys/vicuna-7b-v1.5', 'lmsys/vicuna-13b-v1.5', + 'meta-llama/Llama-2-7b-chat-hf' +] +Auto_model = ['chatglm'] + + +class HFChatModel: + + def _get_context_length(self, model, model_path): + # By default, we use model.config.seq_length + model_path = model_path.lower() + if 'baichuan' in model_path: + context_window = model.config.model_max_length + elif 'internlm' in model_path or 'llama' in model_path: + context_window = model.config.max_position_embeddings + elif 'vicuna' in model_path: + context_window = model.generation_config.max_length + else: + # chatglm & qwen + context_window = model.config.seq_length + return context_window + + def _get_context_length_robust(self, model, model_path): + try: + context_window = self._get_context_length(model, model_path) + return context_window + except: + self.logger.critical( + 'Failed to extract context_window information from config / generation_config. ' + 'Please read the above code and check if the logic works for you model path' + ) + raise NotImplementedError + + def __init__(self, + model_path, + system_prompt: str = None, + **kwargs): + + self.logger = get_logger('HFChatModel') + if 'vicuna' in model_path.lower(): + try: + from fastchat.model import get_conversation_template + except: + self.logger.critical('Please install fastchat first to use vicuna. ') + sys.exit(-1) + + self.explicit_device = kwargs.pop('device', None) + + if self.explicit_device is None: + # If CUDA_VISIBLE_DEVICES is not properly set + if 'CUDA_VISIBLE_DEVICES' not in os.environ or os.environ['CUDA_VISIBLE_DEVICES'] == '0,1,2,3,4,5,6,7': + num_gpu = get_gpu_num(model_path) + gpu_offset = kwargs.pop('gpu_offset', 0) + cuda_visible_devices = ','.join([str(i) for i in range(gpu_offset, gpu_offset + num_gpu)]) + os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices + + from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModel + from transformers.generation import GenerationConfig + + if model_path not in validated_llms: + self.logger.warning(f'{model_path} not in validated LLMs, may have inference troubles. ') + + self.model_path = model_path + if listinstr(Auto_model, model_path): + LoadModel = AutoModel + else: + LoadModel = AutoModelForCausalLM + + assert osp.exists(model_path) or len(model_path.split('/')) == 2 + + device = self.explicit_device if self.explicit_device else 'auto' + + precision = {} + if 'internlm-chat-7b' in model_path: + precision = {'torch_dtype': torch.float16} + elif 'internlm-chat-20b' in model_path: + precision = {'torch_dtype': torch.bfloat16} + + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = LoadModel.from_pretrained(model_path, trust_remote_code=True, device_map='cpu', **precision) + model = model.eval() + + if device != 'cpu': + model = model.to(f'cuda:{device}' if isinstance(device, int) else 'cuda') + try: + model.generation_config = GenerationConfig.from_pretrained( + model_path, trust_remote_code=True, device_map=device) + except: + pass + + torch.cuda.empty_cache() + self.model = model + self.context_length = self._get_context_length_robust(model=model, model_path=model_path) + self.answer_buffer = 192 + self.system_prompt = system_prompt + for k, v in kwargs.items(): + self.logger.info(f'Following args will be used for generation (If not set specifically), {k}: {v}. ') + self.kwargs = kwargs + + def generate_str(self, input, **kwargs): + if 'baichuan' in self.model_path.lower(): + messages = [] + messages.append({'role': 'user', 'content': input}) + resp = self.model.chat(self.tokenizer, messages, **kwargs) + elif 'vicuna' in self.model_path.lower(): + from fastchat.model import get_conversation_template + conv = get_conversation_template('vicuna') + conv.append_message(conv.roles[0], input) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + inputs = self.tokenizer([prompt], return_tensors='pt') + if torch.cuda.is_available(): + for k in inputs: + inputs[k] = inputs[k].cuda() + + params = dict(do_sample=True, temperature=0.7, repetition_penalty=1.0, max_new_tokens=512) + params.update(self.kwargs) + params.update(kwargs) + outputs = self.model.generate(**inputs, **params) + resp = self.tokenizer.decode( + outputs[0][len(inputs['input_ids'][0]):], + skip_special_tokens=True, + spaces_between_special_tokens=False) + + else: + params = self.kwargs + params.update(kwargs) + resp, _ = self.model.chat(self.tokenizer, input, history=[], **params) + + return resp + + def length_ok(self, inputs): + tot = len(self.tokenizer.encode(self.system_prompt)) if self.system_prompt is not None else 0 + for s in inputs: + tot += len(self.tokenizer.encode(s)) + return tot + self.answer_buffer < self.context_length + + def generate_list(self, full_inputs, offset=0, **kwargs): + assert isinstance(full_inputs, list) + + inputs = full_inputs[offset:] + if not self.length_ok(inputs): + return self.chat(full_inputs, offset + 1) + + model_path = self.model_path.lower() + + if sum([x in model_path for x in ['baichuan']]): + input_msgs = [] + if self.system_prompt is not None: + input_msgs.append(dict(role='user', content=self.system_prompt)) + if len(inputs): + assert isinstance(inputs, list) and isinstance(inputs[0], str) + roles = ['user', 'assistant'] if len(inputs) % 2 == 1 else ['assistant', 'user'] + roles = roles * len(inputs) + for role, msg in zip(roles, inputs): + input_msgs.append(dict(role=role, content=msg)) + response = self.model.chat(self.tokenizer, input_msgs) + elif sum([x in model_path for x in ['vicuna']]): + from fastchat.model import get_conversation_template + conv = get_conversation_template('vicuna') + assert isinstance(inputs, list) and isinstance(inputs[0], str) + if len(inputs) % 2 == 1: + if self.system_prompt is not None: + conv.append_message(conv.roles[0], self.system_prompt) + for i in range(len(inputs) // 2): + conv.append_message(conv.roles[0], inputs[2 * i]) + conv.append_message(conv.roles[1], inputs[2 * i + 1]) + else: + assert self.system_prompt is not None + conv.append_message(conv.roles[0], self.system_prompt) + conv.append_message(conv.roles[1], inputs[0]) + for i in range(len(inputs) // 2 - 1): + conv.append_message(conv.roles[0], inputs[2 * i + 1]) + conv.append_message(conv.roles[1], inputs[2 * i + 2]) + conv.append_message(conv.roles[0], inputs[-1]) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + inputs = self.tokenizer([prompt], return_tensors='pt') + if torch.cuda.is_available(): + for k in inputs: + inputs[k] = inputs[k].cuda() + + params = dict(do_sample=True, temperature=0.7, repetition_penalty=1.0, max_new_tokens=512) + params.update(self.kwargs) + params.update(kwargs) + + outputs = self.model.generate(**inputs, **params) + response = self.tokenizer.decode( + outputs[0][len(inputs['input_ids'][0]):], + skip_special_tokens=True, + spaces_between_special_tokens=False) + response = response.lstrip('\n') + else: + # The default option, support internlm, chatglm, qwen + history, msg = [], None + if len(inputs) % 2 == 1: + if self.system_prompt is not None: + history = [(self.system_prompt, '')] + for i in range(len(inputs) // 2): + history.append((inputs[2 * i], inputs[2 * i + 1])) + else: + assert self.system_prompt is not None + history = [(self.system_prompt, inputs[0])] + for i in range(len(inputs) // 2 - 1): + history.append((inputs[2 * i + 1], inputs[2 * i + 2])) + msg = inputs[-1] + + params = self.kwargs + params.update(kwargs) + response, _ = self.model.chat(self.tokenizer, msg, history=history, **params) + + return response, offset + + def generate(self, inputs, **kwargs): + if isinstance(inputs, str): + return self.generate_str(inputs, **kwargs) + elif isinstance(inputs, list): + return self.generate_list(inputs, **kwargs) diff --git a/vlmeval/api/qwen_api.py b/vlmeval/api/qwen_api.py new file mode 100644 index 0000000..4e490d5 --- /dev/null +++ b/vlmeval/api/qwen_api.py @@ -0,0 +1,75 @@ +from http import HTTPStatus +import os +from vlmeval.api.base import BaseAPI +from vlmeval.smp import * + + +# Note: This is a pure language model API. +class QwenAPI(BaseAPI): + + is_api: bool = True + + def __init__(self, + model: str = 'qwen-max-1201', + retry: int = 5, + wait: int = 5, + verbose: bool = True, + seed: int = 2680, + temperature: float = 0.0, + system_prompt: str = None, + key: str = None, + max_tokens: int = 1024, + proxy: str = None, + **kwargs): + + assert model in ['qwen-turbo', 'qwen-plus', 'qwen-max', 'qwen-max-1201', 'qwen-max-longcontext'] + self.model = model + import dashscope + self.fail_msg = 'Failed to obtain answer via API. ' + self.max_tokens = max_tokens + self.temperature = temperature + self.seed = seed + if key is None: + key = os.environ.get('DASHSCOPE_API_KEY', None) + assert key is not None, ( + 'Please set the API Key (obtain it here: ' + 'https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start)' + ) + dashscope.api_key = key + if proxy is not None: + proxy_set(proxy) + super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs) + + @staticmethod + def build_msgs(msgs_raw, system_prompt=None): + msgs = cp.deepcopy(msgs_raw) + ret = [] + if system_prompt is not None: + ret.append(dict(role='system', content=system_prompt)) + for i, msg in enumerate(msgs): + role = 'user' if i % 2 == 0 else 'assistant' + ret.append(dict(role=role, content=msg)) + return ret + + def generate_inner(self, inputs, **kwargs) -> str: + from dashscope import MultiModalConversation + assert isinstance(inputs, str) or isinstance(inputs, list) + inputs = [inputs] if isinstance(inputs, str) else inputs + messages = self.build_msgs(msgs_raw=inputs, system_prompt=self.system_prompt) + + import dashscope + response = dashscope.Generation.call( + model=self.model, + messages=messages, + seed=self.seed, + temperature=self.temperature, + max_tokens=self.max_tokens, + result_format='message', # set the result to be "message" format. + ) + if response.status_code != HTTPStatus.OK: + return -1, 'Error: Bad Response Statuse Code. ', f'The response status code is {response.status_code}. ' + + try: + return 0, response['output']['choices'][0]['message']['content'].strip(), 'Succeeded! ' + except Exception as err: + return -1, f'Error: Failed to parse the response. {err}', response diff --git a/vlmeval/api/qwen_vl_api.py b/vlmeval/api/qwen_vl_api.py new file mode 100644 index 0000000..131c89a --- /dev/null +++ b/vlmeval/api/qwen_vl_api.py @@ -0,0 +1,79 @@ +from vlmeval.smp import * +from vlmeval.api.base import BaseAPI + + +class QwenVLWrapper(BaseAPI): + + is_api: bool = True + + def __init__(self, + model: str = 'qwen-vl-plus', + retry: int = 5, + wait: int = 5, + key: str = None, + verbose: bool = True, + temperature: float = 0.0, + system_prompt: str = None, + max_tokens: int = 1024, + proxy: str = None, + **kwargs): + + assert model in ['qwen-vl-plus', 'qwen-vl-max'] + self.model = model + import dashscope + self.fail_msg = 'Failed to obtain answer via API. ' + self.max_tokens = max_tokens + self.temperature = temperature + if key is None: + key = os.environ.get('DASHSCOPE_API_KEY', None) + assert key is not None, ( + 'Please set the API Key (obtain it here: ' + 'https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start)' + ) + dashscope.api_key = key + if proxy is not None: + proxy_set(proxy) + super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs) + + @staticmethod + def build_msgs(msgs_raw, system_prompt=None): + msgs = cp.deepcopy(msgs_raw) + ret = [] + if system_prompt is not None: + content = list(dict(text=system_prompt)) + ret.append(dict(role='system', content=content)) + content = [] + for msg in msgs: + if msg['type'] == 'text': + content.append(dict(text=msg['value'])) + elif msg['type'] == 'image': + content.append(dict(image='file://' + msg['value'])) + ret.append(dict(role='user', content=content)) + return ret + + def generate_inner(self, inputs, **kwargs) -> str: + from dashscope import MultiModalConversation + assert isinstance(inputs, str) or isinstance(inputs, list) + pure_text = np.all([x['type'] == 'text' for x in inputs]) + assert not pure_text + messages = self.build_msgs(msgs_raw=inputs, system_prompt=self.system_prompt) + gen_config = dict(max_output_tokens=self.max_tokens, temperature=self.temperature) + gen_config.update(kwargs) + try: + response = MultiModalConversation.call(model=self.model, messages=messages) + if self.verbose: + print(response) + answer = response.output.choices[0]['message']['content'][0]['text'] + return 0, answer, 'Succeeded! ' + except Exception as err: + if self.verbose: + self.logger.error(err) + self.logger.error(f'The input messages are {inputs}.') + + return -1, '', '' + + +class QwenVLAPI(QwenVLWrapper): + + def generate(self, message, dataset=None): + return super(QwenVLAPI, self).generate(message) diff --git a/vlmeval/api/stepai.py b/vlmeval/api/stepai.py new file mode 100644 index 0000000..f1c3dd1 --- /dev/null +++ b/vlmeval/api/stepai.py @@ -0,0 +1,85 @@ +from vlmeval.smp import * +from vlmeval.api.base import BaseAPI + +url = 'https://b-openapi.basemind.com/openapi/v1/chat/completions' +headers = { + 'X-Request-Orgcode': 'companyA', + 'Authorization': 'Bearer {}', + 'Content-Type': 'application/json' +} + + +class StepAPI(BaseAPI): + + is_api: bool = True + + def __init__(self, + model: str = 'stepapi-rankboard', + retry: int = 10, + wait: int = 3, + key: str = None, + temperature: float = 0, + max_tokens: int = 300, + verbose: bool = True, + system_prompt: str = None, + **kwargs): + self.model = model + self.fail_msg = 'Fail to obtain answer via API.' + self.headers = headers + self.temperature = temperature + self.max_tokens = max_tokens + self.system_prompt = system_prompt + if key is not None: + self.key = key + else: + self.key = os.environ.get('STEPAI_API_KEY', '') + headers['Authorization'] = headers['Authorization'].format(self.key) + + super().__init__(retry=retry, wait=wait, verbose=verbose, system_prompt=system_prompt, **kwargs) + + @staticmethod + def build_msgs(msgs_raw): + messages = [] + message = {'role': 'user', 'content': []} + + for msg in msgs_raw: + if msg['type'] == 'image': + image_b64 = encode_image_file_to_base64(msg['value']) + message['content'].append({ + 'image_b64': {'b64_json': image_b64}, + 'type': 'image_b64' + }) + elif msg['type'] == 'text': + message['content'].append({ + 'text': msg['value'], + 'type': 'text' + }) + + messages.append(message) + return messages + + def generate_inner(self, inputs, **kwargs) -> str: + print(inputs, '\n') + payload = dict( + model=self.model, + max_tokens=self.max_tokens, + temperature=self.temperature, + messages=self.build_msgs(msgs_raw=inputs), + **kwargs) + response = requests.post(url, headers=headers, data=json.dumps(payload)) + ret_code = response.status_code + ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code + + answer = self.fail_msg + try: + resp_struct = json.loads(response.text) + answer = resp_struct['choices'][0]['message']['content'].strip() + except: + pass + return ret_code, answer, response + + +class Step1V(StepAPI): + + def generate(self, message, dataset=None): + return super(StepAPI, self).generate(message) diff --git a/vlmeval/config.py b/vlmeval/config.py new file mode 100644 index 0000000..9c0f129 --- /dev/null +++ b/vlmeval/config.py @@ -0,0 +1,137 @@ +from vlmeval.vlm import * +from vlmeval.api import * +from functools import partial + +PandaGPT_ROOT = None +MiniGPT4_ROOT = None +TransCore_ROOT = None +Yi_ROOT = None +OmniLMM_ROOT = None +LLAVA_V1_7B_MODEL_PTH = 'Please set your local path to LLaVA-7B-v1.1 here, the model weight is obtained by merging LLaVA delta weight based on vicuna-7b-v1.1 in https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md with vicuna-7b-v1.1. ' + +ungrouped = { + 'TransCore_M': partial(TransCoreM, root=TransCore_ROOT), + 'PandaGPT_13B': partial(PandaGPT, name='PandaGPT_13B', root=PandaGPT_ROOT), + 'flamingov2': partial(OpenFlamingo, name='v2', mpt_pth='anas-awadalla/mpt-7b', ckpt_pth='openflamingo/OpenFlamingo-9B-vitl-mpt7b'), + 'VisualGLM_6b': partial(VisualGLM, model_path='THUDM/visualglm-6b'), + 'mPLUG-Owl2': partial(mPLUG_Owl2, model_path='MAGAer13/mplug-owl2-llama2-7b'), + 'cogvlm-grounding-generalist':partial(CogVlm, name='cogvlm-grounding-generalist',tokenizer_name ='lmsys/vicuna-7b-v1.5'), + 'cogvlm-chat':partial(CogVlm, name='cogvlm-chat',tokenizer_name ='lmsys/vicuna-7b-v1.5'), + 'emu2_chat':partial(Emu, model_path='BAAI/Emu2-Chat',), + 'MMAlaya':partial(MMAlaya, model_path='DataCanvas/MMAlaya'), + 'MiniCPM-V':partial(MiniCPM_V, model_path='openbmb/MiniCPM-V'), + 'MiniCPM-V-2':partial(MiniCPM_V, model_path='openbmb/MiniCPM-V-2'), + 'OmniLMM_12B':partial(OmniLMM12B, model_path='openbmb/OmniLMM-12B', root=OmniLMM_ROOT), +} + +api_models = { + 'GPT4V': partial(GPT4V, model='gpt-4-1106-vision-preview', temperature=0, img_size=512, img_detail='low', retry=10), + 'GPT4V_HIGH': partial(GPT4V, model='gpt-4-1106-vision-preview', temperature=0, img_size=-1, img_detail='high', retry=10), + 'GPT4V_20240409': partial(GPT4V, model='gpt-4-turbo-2024-04-09', temperature=0, img_size=512, img_detail='low', retry=10), + 'GPT4V_20240409_HIGH': partial(GPT4V, model='gpt-4-turbo-2024-04-09', temperature=0, img_size=-1, img_detail='high', retry=10), + 'GeminiProVision': partial(GeminiProVision, temperature=0, retry=10), + 'QwenVLPlus': partial(QwenVLAPI, model='qwen-vl-plus', temperature=0, retry=10), + 'QwenVLMax': partial(QwenVLAPI, model='qwen-vl-max', temperature=0, retry=10), + # Internal Only + 'GPT4V_INT': partial(GPT4V_Internal, model='gpt-4-vision-preview', temperature=0, img_size=512, img_detail='low', retry=10), + 'Step1V': partial(Step1V, temperature=0, retry=10), + 'Claude3V_Opus': partial(Claude3V, model='claude-3-opus-20240229', temperature=0, retry=10), + 'Claude3V_Sonnet': partial(Claude3V, model='claude-3-sonnet-20240229', temperature=0, retry=10), + 'Claude3V_Haiku': partial(Claude3V, model='claude-3-haiku-20240307', temperature=0, retry=10), +} + +xtuner_series = { + 'llava-internlm2-7b': partial(LLaVA_XTuner, llm_path='internlm/internlm2-chat-7b', llava_path='xtuner/llava-internlm2-7b', visual_select_layer=-2, prompt_template='internlm2_chat'), + 'llava-internlm2-20b': partial(LLaVA_XTuner, llm_path='internlm/internlm2-chat-20b', llava_path='xtuner/llava-internlm2-20b', visual_select_layer=-2, prompt_template='internlm2_chat'), + 'llava-internlm-7b': partial(LLaVA_XTuner, llm_path='internlm/internlm-chat-7b', llava_path='xtuner/llava-internlm-7b', visual_select_layer=-2, prompt_template='internlm_chat'), + 'llava-v1.5-7b-xtuner': partial(LLaVA_XTuner, llm_path='lmsys/vicuna-7b-v1.5', llava_path='xtuner/llava-v1.5-7b-xtuner', visual_select_layer=-2, prompt_template='vicuna'), + 'llava-v1.5-13b-xtuner': partial(LLaVA_XTuner, llm_path='lmsys/vicuna-13b-v1.5', llava_path='xtuner/llava-v1.5-13b-xtuner', visual_select_layer=-2, prompt_template='vicuna'), +} + +qwen_series = { + 'qwen_base': partial(QwenVL, model_path='Qwen/Qwen-VL'), + 'qwen_chat': partial(QwenVL, model_path='Qwen/Qwen-VL-Chat'), + 'monkey':partial(Monkey, model_path='echo840/Monkey'), + 'monkey-chat':partial(MonkeyChat, model_path='echo840/Monkey-Chat') +} + +llava_series = { + 'llava_v1.5_7b': partial(LLaVA, model_pth='liuhaotian/llava-v1.5-7b'), + 'llava_v1.5_13b': partial(LLaVA, model_pth='liuhaotian/llava-v1.5-13b'), + 'llava_v1_7b': partial(LLaVA, model_pth=LLAVA_V1_7B_MODEL_PTH), + 'sharegpt4v_7b': partial(LLaVA, model_pth='Lin-Chen/ShareGPT4V-7B'), + 'sharegpt4v_13b': partial(LLaVA, model_pth='Lin-Chen/ShareGPT4V-13B'), + 'llava_next_vicuna_7b': partial(LLaVA_Next, model_pth='llava-hf/llava-v1.6-vicuna-7b-hf'), + 'llava_next_vicuna_13b': partial(LLaVA_Next, model_pth='llava-hf/llava-v1.6-vicuna-13b-hf'), + 'llava_next_mistral_7b': partial(LLaVA_Next, model_pth='llava-hf/llava-v1.6-mistral-7b-hf'), + 'llava_next_yi_34b': partial(LLaVA_Next, model_pth='llava-hf/llava-v1.6-34b-hf'), +} + +internvl_series = { + 'InternVL-Chat-V1-1':partial(InternVLChat, model_path='OpenGVLab/InternVL-Chat-Chinese-V1-1'), + 'InternVL-Chat-V1-2': partial(InternVLChat, model_path='OpenGVLab/InternVL-Chat-Chinese-V1-2'), + 'InternVL-Chat-V1-2-Plus': partial(InternVLChat, model_path='OpenGVLab/InternVL-Chat-Chinese-V1-2-Plus'), + 'InternVL-Chat-V1-5': partial(InternVLChat, model_path='OpenGVLab/InternVL-Chat-V1-5-Preview'), +} + +yivl_series = { + 'Yi_VL_6B':partial(Yi_VL, model_path='01-ai/Yi-VL-6B', root=Yi_ROOT), + 'Yi_VL_34B':partial(Yi_VL, model_path='01-ai/Yi-VL-34B', root=Yi_ROOT), +} + +xcomposer_series = { + 'XComposer': partial(XComposer, model_path='internlm/internlm-xcomposer-vl-7b'), + 'sharecaptioner': partial(ShareCaptioner, model_path='Lin-Chen/ShareCaptioner'), + 'XComposer2': partial(XComposer2, model_path='internlm/internlm-xcomposer2-vl-7b'), + 'XComposer2_1.8b': partial(XComposer2, model_path='internlm/internlm-xcomposer2-vl-1_8b'), + 'XComposer2_4KHD': partial(XComposer2_4KHD, model_path='internlm/internlm-xcomposer2-4khd-7b'), +} + +minigpt4_series = { + 'MiniGPT-4-v2': partial(MiniGPT4, mode='v2', root=MiniGPT4_ROOT), + 'MiniGPT-4-v1-7B': partial(MiniGPT4, mode='v1_7b', root=MiniGPT4_ROOT), + 'MiniGPT-4-v1-13B': partial(MiniGPT4, mode='v1_13b', root=MiniGPT4_ROOT), +} + +idefics_series = { + 'idefics_9b_instruct': partial(IDEFICS, model_pth='HuggingFaceM4/idefics-9b-instruct'), + 'idefics_80b_instruct': partial(IDEFICS, model_pth='HuggingFaceM4/idefics-80b-instruct'), + 'idefics2_8b': partial(IDEFICS2, model_path='HuggingFaceM4/idefics2-8b'), +} + +instructblip_series = { + 'instructblip_7b': partial(InstructBLIP, name='instructblip_7b'), + 'instructblip_13b': partial(InstructBLIP, name='instructblip_13b'), +} + +deepseekvl_series = { + 'deepseek_vl_7b': partial(DeepSeekVL, model_path='deepseek-ai/deepseek-vl-7b-chat'), + 'deepseek_vl_1.3b': partial(DeepSeekVL, model_path='deepseek-ai/deepseek-vl-1.3b-chat'), +} + +supported_VLM = {} + +model_groups = [ + ungrouped, api_models, + xtuner_series, qwen_series, llava_series, internvl_series, yivl_series, + xcomposer_series, minigpt4_series, idefics_series, instructblip_series, + deepseekvl_series +] + +for grp in model_groups: + supported_VLM.update(grp) + +transformer_ver = {} +transformer_ver['4.33.0'] = list(qwen_series) + list(internvl_series) + list(xcomposer_series) + [ + 'mPLUG-Owl2', 'flamingov2', 'VisualGLM_6b', 'MMAlaya', 'PandaGPT_13B' +] + list(idefics_series) + list(minigpt4_series) + list(instructblip_series) +transformer_ver['4.37.0'] = [x for x in llava_series if 'next' not in x] + [ + 'TransCore_M', 'cogvlm-chat', 'cogvlm-grounding-generalist', 'emu2_chat', 'MiniCPM-V', 'MiniCPM-V-2', 'OmniLMM_12B' +] + list(xtuner_series) + list(yivl_series) + list(deepseekvl_series) +transformer_ver['4.39.0'] = [x for x in llava_series if 'next' in x] + +if __name__ == '__main__': + import sys + ver = sys.argv[1] + if ver in transformer_ver: + print(' '.join(transformer_ver[ver])) \ No newline at end of file diff --git a/vlmeval/evaluate/OCRBench.py b/vlmeval/evaluate/OCRBench.py new file mode 100644 index 0000000..c37ad08 --- /dev/null +++ b/vlmeval/evaluate/OCRBench.py @@ -0,0 +1,65 @@ +from vlmeval.smp import * + + +def OCRBench_eval(eval_file): + OCRBench_score = { + 'Regular Text Recognition': 0, + 'Irregular Text Recognition': 0, + 'Artistic Text Recognition': 0, + 'Handwriting Recognition': 0, + 'Digit String Recognition': 0, + 'Non-Semantic Text Recognition': 0, + 'Scene Text-centric VQA': 0, + 'Doc-oriented VQA': 0, + 'Key Information Extraction': 0, + 'Handwritten Mathematical Expression Recognition': 0 + } + + logger = get_logger('Evaluation') + + data = load(eval_file) + lt = len(data) + lines = [data.iloc[i] for i in range(lt)] + for i in tqdm(range(len(lines))): + line = lines[i] + predict = str(line['prediction']) + answers = eval(line['answer']) + category = line['category'] + if category == 'Handwritten Mathematical Expression Recognition': + for j in range(len(answers)): + answer = answers[j].strip().replace('\n', ' ').replace(' ', '') + predict = predict.strip().replace('\n', ' ').replace(' ', '') + if answer in predict: + OCRBench_score[category] += 1 + break + else: + for j in range(len(answers)): + answer = answers[j].lower().strip().replace('\n', ' ') + predict = predict.lower().strip().replace('\n', ' ') + if answer in predict: + OCRBench_score[category] += 1 + break + + final_score_dict = {} + final_score_dict['Text Recognition'] = ( + OCRBench_score['Regular Text Recognition'] + OCRBench_score['Irregular Text Recognition'] + + OCRBench_score['Artistic Text Recognition'] + OCRBench_score['Handwriting Recognition'] + + OCRBench_score['Digit String Recognition'] + OCRBench_score['Non-Semantic Text Recognition'] + ) + final_score_dict['Scene Text-centric VQA'] = OCRBench_score['Scene Text-centric VQA'] + final_score_dict['Doc-oriented VQA'] = OCRBench_score['Doc-oriented VQA'] + final_score_dict['Key Information Extraction'] = OCRBench_score['Key Information Extraction'] + final_score_dict['Handwritten Mathematical Expression Recognition'] = \ + OCRBench_score['Handwritten Mathematical Expression Recognition'] + final_score_dict['Final Score'] = ( + final_score_dict['Text Recognition'] + final_score_dict['Scene Text-centric VQA'] + + final_score_dict['Doc-oriented VQA'] + final_score_dict['Key Information Extraction'] + + final_score_dict['Handwritten Mathematical Expression Recognition'] + ) + final_score_dict['Final Score Norm'] = float(final_score_dict['Final Score']) / 10 + score_pth = eval_file.replace('.xlsx', '_score.json') + dump(final_score_dict, score_pth) + logger.info(f'OCRBench_eval successfully finished evaluating {eval_file}, results saved in {score_pth}') + logger.info('Score: ') + for key, value in final_score_dict.items(): + logger.info('{}:{}'.format(key, value)) diff --git a/vlmeval/evaluate/__init__.py b/vlmeval/evaluate/__init__.py new file mode 100644 index 0000000..10248c4 --- /dev/null +++ b/vlmeval/evaluate/__init__.py @@ -0,0 +1,9 @@ +from .yes_or_no import default_rating, MME_rating, YOrN_eval +from .mmvet_eval import MMVet_eval +from .multiple_choice import multiple_choice_eval +from .coco_eval import COCO_eval +from .vqa_eval import VQAEval +from .mathvista_eval import MathVista_eval +from .llavabench import LLaVABench_eval +from .misc import build_judge +from .OCRBench import OCRBench_eval diff --git a/vlmeval/evaluate/coco_eval.py b/vlmeval/evaluate/coco_eval.py new file mode 100644 index 0000000..9ae8f4f --- /dev/null +++ b/vlmeval/evaluate/coco_eval.py @@ -0,0 +1,74 @@ +from vlmeval.smp import * +from pycocoevalcap.bleu.bleu import Bleu +from pycocoevalcap.rouge.rouge import Rouge +from pycocoevalcap.cider.cider import Cider + + +class COCO_Caption_Scorer(): + def __init__(self, ref, gt): + self.ref = ref + self.gt = gt + print('setting up scorers...') + self.scorers = [ + (Bleu(4), ['Bleu_1', 'Bleu_2', 'Bleu_3', 'Bleu_4']), + # (Meteor(), "METEOR"), # need java version 11.0.16+ + (Rouge(), 'ROUGE_L'), + (Cider(), 'CIDEr'), + # (Spice(), "SPICE"), # need java version 11.0.16+ + ] + + def compute_scores(self): + total_scores = {} + for scorer, method in self.scorers: + print('computing %s score...' % (scorer.method())) + score, scores = scorer.compute_score(self.gt, self.ref) + if type(method) == list: + for sc, scs, m in zip(score, scores, method): + print('%s: %0.3f' % (m, sc * 100)) + total_scores['Bleu'] = [x * 100 for x in score] + else: + print('%s: %0.3f' % (method, score * 100)) + total_scores[method] = score * 100 + + print('*****DONE*****') + for key, value in total_scores.items(): + print('{}:{}'.format(key, value)) + return total_scores + + +def COCO_eval(eval_file, nproc=4, verbose=False): + logger = get_logger('Evaluation') + + data = load(eval_file) + + lt = len(data) + lines = [data.iloc[i] for i in range(lt)] + ref = {} + gt = {} + for i, line in enumerate(lines): + ref[str(i)] = [str(line['prediction'])] + gt[str(i)] = eval(line['answer']) + + scorer = COCO_Caption_Scorer(ref, gt) + coco_caption_score_dict = scorer.compute_scores() + + score_pth = eval_file.replace('.xlsx', '_score.json') + dump(coco_caption_score_dict, score_pth) + logger.info(f'COCO_eval successfully finished evaluating {eval_file}, results saved in {score_pth}') + logger.info('Score: ') + for key, value in coco_caption_score_dict.items(): + logger.info('{}:{}'.format(key, value)) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Inference LLM Answers. ') + parser.add_argument('--data', type=str, help='The question set for inference, in excel / tsv / json format. ') + parser.add_argument('--nproc', type=int, default=4) + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + COCO_eval(eval_file=args.data, nproc=args.nproc, verbose=args.verbose) diff --git a/vlmeval/evaluate/llavabench.py b/vlmeval/evaluate/llavabench.py new file mode 100644 index 0000000..c44b203 --- /dev/null +++ b/vlmeval/evaluate/llavabench.py @@ -0,0 +1,113 @@ +import argparse +import numpy as np +import pandas as pd +import os.path as osp +from vlmeval.evaluate.misc import build_judge +from vlmeval.smp import get_logger, load, dump, defaultdict +from vlmeval.utils import track_progress_rich + +rule_dict = { + 'llava_bench_conv': {'role': 'Assistant', 'prompt': 'We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment.'}, # noqa: E501 + 'llava_bench_detail': {'role': 'Assistant', 'prompt': 'We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment.'}, # noqa: E501 + 'llava_bench_complex': {'role': 'Assistant', 'prompt': 'We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment.'} # noqa: E501 +} + + +def get_eval(judge, content): + return judge.generate(content) + + +def parse_score(review): + logger = get_logger('Evaluation') + try: + score_pair = review.split('\n')[0] + score_pair = score_pair.replace(',', ' ') + sp = score_pair.split(' ') + if len(sp) == 2: + return [float(sp[0]), float(sp[1])] + else: + logger.error('error', review) + return [-1, -1] + except Exception as e: + logger.error(e, 'error', review) + return [-1, -1] + + +def build_prompt(line): + cap_str = line['caption'] + question = line['question'] + ans1 = line['gpt4_ans'] + ans2 = line['prediction'] + category = 'llava_bench_' + line['category'] + rule = rule_dict[category] + role, prompt = rule['role'], rule['prompt'] + + content = (f'[Context]\n{cap_str}\n\n' + f'[Question]\n{question}\n\n' + f'[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n' + f'[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n' + f'[System]\n{prompt}\n\n') + return content + + +def LLaVABench_atomeval(model, prompt): + review = get_eval(model, prompt) + scores = parse_score(review) + return scores + + +def LLaVABench_score(data): + cates = ['overall'] + list(set(data['category'])) + ret = defaultdict(list) + + for c in cates: + ret['split'].append(c) + sub = data[data['category'] == c] if c != 'overall' else data + ret['Relative Score (main)'].append(np.mean(sub['score']) / np.mean(sub['gpt4_score']) * 100) + ret['VLM Score'].append(np.mean(sub['score']) * 10) + ret['GPT4 Score'].append(np.mean(sub['gpt4_score']) * 10) + return pd.DataFrame(ret) + + +def LLaVABench_eval(eval_file, **judge_kwargs): + suffix = '.' + eval_file.split('.')[-1] + record_file = eval_file.replace(suffix, '_openai_result' + suffix) + score_file = eval_file.replace(suffix, '_score.csv') + nproc = judge_kwargs.pop('nproc', 4) + + if not osp.exists(record_file): + data = load(eval_file) + lines = [data.iloc[i] for i in range(len(data))] + model = build_judge( + temperature=0.2, + system_prompt='You are a helpful and precise assistant for checking the quality of the answer.', + **judge_kwargs) + prompts = [build_prompt(line) for line in lines] + tups = [(model, prompt) for prompt in prompts] + scores = track_progress_rich(LLaVABench_atomeval, tups, nproc=nproc, chunksize=nproc) + data['gpt4_score'] = [x[0] for x in scores] + data['score'] = [x[1] for x in scores] + dump(data, record_file) + + data = load(record_file) + ret = LLaVABench_score(data).round(1) + print(ret) + dump(ret, score_file) + return ret + + +def parse_args(): + parser = argparse.ArgumentParser(description='LLaVABench Evaluation. ') + parser.add_argument('data', type=str, help='The question set for inference, in excel / tsv / json format. ') + parser.add_argument( + '--model', type=str, help='The LLM (GPT) used for inference. ', default='gpt-4-turbo', + choices=['gpt-4-0613', 'gpt-4-turbo', 'chatgpt-1106', 'chatgpt-0613', 'gpt-4-0314']) + parser.add_argument('--nproc', type=int, default=4) + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + LLaVABench_eval(eval_file=args.data, model=args.model, nproc=args.nproc, verbose=args.verbose) diff --git a/vlmeval/evaluate/mathvista_eval.py b/vlmeval/evaluate/mathvista_eval.py new file mode 100644 index 0000000..b84e967 --- /dev/null +++ b/vlmeval/evaluate/mathvista_eval.py @@ -0,0 +1,234 @@ +from vlmeval.evaluate.misc import build_judge +from vlmeval.smp import * +from vlmeval.utils import track_progress_rich +from vlmeval.utils.matching_util import can_infer + + +def get_gpt4_ICE(): + example_1 = """ +Hint: Please answer the question requiring an integer answer and provide the final value, +e.g., 1, 2, 3, at the end.\n +Question: Which number is missing?\n +Model response: The number missing in the sequence is 14.\n +Extracted answer: 14 +""" + + example_2 = """ +Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, +e.g., 1.2, 1.3, 1.4, at the end.\n +Question: What is the fraction of females facing the camera?\n +Model response: The fraction of females facing the camera is 0.6, +which means that six out of ten females in the group are facing the camera.\n +Extracted answer: 0.6 +""" + + example_3 = """ +Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, +e.g., 1.23, 1.34, 1.45, at the end.\n +Question: How much money does Luca need to buy a sour apple candy and a butter-scotch candy? (Unit: $)\n +Model response: Luca needs $1.45 to buy a sour apple candy and a butterscotch candy.\n +Extracted answer: 1.45 +""" + + example_4 = """ +Hint: Please answer the question requiring a Python list as an answer and provide the final list, +e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end.\n +Question: Between which two years does the line graph saw its maximum peak?\n +Model response: The line graph saw its maximum peak between 2007 and 2008.\n +Extracted answer: [2007, 2008] +""" + + example_5 = """ +Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end.\n +Question: What fraction of the shape is blue?\n +Choices: (A) 3/11 (B) 8/11 (C) 6/11 (D) 3/5\n +Model response: The correct answer is (B) 8/11.\n +Extracted answer: B +""" + + return [example_1, example_2, example_3, example_4, example_5] + + +def build_mathvista_gpt4_prompt(line): + task_description = """ +Please read the following example. +Then extract the answer from the model response and type it at the end of the prompt.\n +""" + question = line['question'] + prediction = str(line['prediction']) + prompt = task_description + examples = get_gpt4_ICE() + for example in examples: + prompt += example + '\n' + prompt += question + '\n' + prompt += 'Model respone: ' + prediction + prompt += 'Extracted answer:' + return prompt + + +def list_to_dict(lst): + return {chr(65 + i): val for i, val in enumerate(lst)} + + +def post_check(line, prefetch=False): + res = None + ans = line['answer'] + response = line['prediction'] if prefetch else line['res'] + try: + if line['question_type'] == 'multi_choice': + ans = line['answer_option'] + choices = list_to_dict(eval(line['choices'])) + res = can_infer(response, choices) + if prefetch: + return res + else: + if line['answer_type'] == 'integer': + res = int(response) + ans = int(line['answer']) + elif line['answer_type'] == 'float': + res = float(response) + ans = float(line['answer']) + else: + res = str(res) + ans = str(ans) + except ValueError: + pass + + if res == ans: + return res if prefetch else True + else: + return False + + +def MathVista_auxeval(model, line): + prompt = build_mathvista_gpt4_prompt(line) + log = '' + retry = 5 + if post_check(line, prefetch=True): + res = post_check(line, prefetch=True) + return dict(log='Prefetch succeed', res=res) + for i in range(retry): + prediction = line['prediction'] + res = model.generate(prompt, temperature=i * 0.5) + if res is None: + log += f'Try {i}: output is {prediction}, failed to parse.\n' + else: + log += 'Succeed' + return dict(log=log, res=res) + log += 'All 5 retries failed.\n' + return dict(log=log, res='') + + +def MathVista_acc(result_file): + data = load(result_file) + tot = defaultdict(lambda: 0) + fetch = defaultdict(lambda: 0) + hit = defaultdict(lambda: 0) + lt = len(data) + skill_list = [] + for i in range(lt): + item = data.iloc[i] + cate = item['task'] + tot['Overall'] += 1 + try: + skills = eval(item['skills']) + except SyntaxError: + skills = [item['skills']] + for skill in skills: + if skill not in skill_list: + skill_list.append(skill) + tot[skill] += 1 + tot[cate] += 1 + if item['log'] == 'Prefetch succeed': + fetch['Overall'] += 1 + fetch[cate] += 1 + for skill in skills: + fetch[skill] += 1 + if post_check(item, prefetch=False): + hit['Overall'] += 1 + hit[cate] += 1 + for skill in skills: + hit[skill] += 1 + + res = defaultdict(list) + for k in tot.keys(): + res['Task&Skill'].append(k) + res['tot'].append(tot[k]) + res['prefetch'].append(fetch[k]) + res['hit'].append(hit[k]) + res['prefetch_rate'].append(fetch[k] / tot[k] * 100) + res['acc'].append(hit[k] / tot[k] * 100) + res = pd.DataFrame(res) + return res + + +def MathVista_eval(eval_file, **judge_kwargs): + logger = get_logger('Evaluation') + model = judge_kwargs['model'] + + suffix = eval_file.split('.')[-1] + storage = eval_file.replace(f'.{suffix}', f'_{model}.xlsx') + tmp_file = eval_file.replace(f'.{suffix}', f'_{model}.pkl') + nproc = judge_kwargs.pop('nproc', 4) + + if osp.exists(storage): + logger.warning(f'GPT scoring file {storage} already exists, will reuse it in MathVista_eval. ') + else: + data = load(eval_file) + model = build_judge(max_tokens=128, **judge_kwargs) + lt = len(data) + lines = [data.iloc[i] for i in range(lt)] + tups = [(model, line) for line in lines] + indices = [line['index'] for line in lines] + + ans = {} + if osp.exists(tmp_file): + ans = load(tmp_file) + tups = [x for x, i in zip(tups, indices) if i not in ans] + indices = [i for i in indices if i not in ans] + + if len(indices): + new_results = track_progress_rich( + MathVista_auxeval, tups, nproc=nproc, chunksize=nproc, + keys=indices, save=tmp_file) + ans = load(tmp_file) + for k, v in zip(indices, new_results): + assert k in ans + assert ans[k]['log'] == v['log'] and ans[k]['res'] == v['res'] + + log_map, res_map = {}, {} + all_inds = [line['index'] for line in lines] + for k in all_inds: + log_map[k] = ans[k]['log'] + res_map[k] = ans[k]['res'] + data['res'] = [res_map[idx] for idx in data['index']] + data['log'] = [log_map[idx] for idx in data['index']] + dump(data, storage) + + score = MathVista_acc(storage) + score_pth = storage.replace('.xlsx', '_score.csv') + + dump(score, score_pth) + logger.info(f'MathVista_eval successfully finished evaluating {eval_file}, results saved in {score_pth}') + logger.info('Score: ') + logger.info(score) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Inference LLM Answers. ') + parser.add_argument('data', type=str, help='The question set for inference, in excel / tsv / json format. ') + parser.add_argument( + '--model', + type=str, + help='The LLM (GPT) used for inference. ', + default='gpt-4-turbo', + choices=['gpt-4-0613', 'gpt-4-turbo', 'chatgpt-1106', 'chatgpt-0613']) + parser.add_argument('--nproc', type=int, default=4) + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + MathVista_eval(eval_file=args.data, model=args.model, nproc=args.nproc, verbose=args.verbose) diff --git a/vlmeval/evaluate/misc.py b/vlmeval/evaluate/misc.py new file mode 100644 index 0000000..ddf0c9e --- /dev/null +++ b/vlmeval/evaluate/misc.py @@ -0,0 +1,29 @@ +import os +from vlmeval.api import OpenAIWrapper, OpenAIWrapperInternal +from vlmeval.smp import load_env + +INTERNAL = os.environ.get('INTERNAL', 0) + + +def build_judge(**kwargs): + model = kwargs.pop('model', None) + load_env() + LOCAL_LLM = os.environ.get('LOCAL_LLM', None) + if LOCAL_LLM is None: + model_map = { + 'gpt-4-turbo': 'gpt-4-1106-preview', + 'gpt-4-0613': 'gpt-4-0613', + 'gpt-4-0314': 'gpt-4-0314', + 'gpt-4-0125': 'gpt-4-0125-preview', + 'chatgpt-1106': 'gpt-3.5-turbo-1106', + 'chatgpt-0613': 'gpt-3.5-turbo-0613', + 'chatgpt-0125': 'gpt-3.5-turbo-0125' + } + model_version = model_map[model] + else: + model_version = LOCAL_LLM + if INTERNAL: + model = OpenAIWrapperInternal(model_version, **kwargs) + else: + model = OpenAIWrapper(model_version, **kwargs) + return model diff --git a/vlmeval/evaluate/mmvet_eval.py b/vlmeval/evaluate/mmvet_eval.py new file mode 100644 index 0000000..0c3868a --- /dev/null +++ b/vlmeval/evaluate/mmvet_eval.py @@ -0,0 +1,185 @@ +from vlmeval.evaluate.misc import build_judge +from vlmeval.smp import * +from vlmeval.utils import track_progress_rich + + +def build_mmvet_gpt4_prompt(line): + question = line['question'] + gt = str(line['answer']) + prediction = str(line['prediction']) + prompt = """ +Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. + in the ground truth means it is totally right +only when all elements in the ground truth are present in the prediction, +and means it is totally right when any one element in the ground truth is present in the prediction. +The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). +Just complete the last space of the correctness score. + +Question | Ground truth | Prediction | Correctness +--- | --- | --- | --- +What is x in the equation? | -1 -5 | x = 3 | 0.0 +What is x in the equation? | -1 -5 | x = -1 | 0.5 +What is x in the equation? | -1 -5 | x = -5 | 0.5 +What is x in the equation? | -1 -5 | x = -5 or 5 | 0.5 +What is x in the equation? | -1 -5 | x = -1 or x = -5 | 1.0 +Can you explain this meme? | This meme is poking fun at the fact that the names of the countries +Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, +while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues +because the names of these countries do not accurately represent their landscapes. | +The meme talks about Iceland and Greenland. It's pointing out that despite their names, +Iceland is not very icy and Greenland isn't very green. | 0.4 +Can you explain this meme? | This meme is poking fun at the fact that the names of the countries +Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, +while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues +because the names of these countries do not accurately represent their landscapes. | +The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. +Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. +The text 'This is why I have trust issues' is a playful way to suggest +that these contradictions can lead to distrust or confusion. +The humor in this meme is derived from the unexpected contrast between the names of the countries +and their actual physical characteristics. | 1.0 +""" + gpt4_prompt = prompt + '\n' + ' | '.join( + [question, gt.replace('', ' ').replace('', ' '), prediction, '']) + return gpt4_prompt + + +def MMVet_auxeval(model, line): + def float_cvt(s): + try: + return float(s) + except ValueError: + return None + + prompt = build_mmvet_gpt4_prompt(line) + log = '' + retry = 5 + for i in range(retry): + output = model.generate(prompt, temperature=i * 0.5) + score = float_cvt(output) + if score is None: + log += f'Try {i}: output is {output}, failed to parse.\n' + elif score < 0 or score > 1: + log += f'Try {i}: output is {output}, invalid score: {score}.\n' + else: + log += 'Succeed' + return dict(log=log, score=score) + log += 'All 5 retries failed.\n' + return dict(log=log, score=0.0) + + +def MMVet_acc(result_file): + data = load(result_file) + tot = defaultdict(lambda: 0) + score = defaultdict(lambda: 0) + lt = len(data) + cate2_list = [] + for i in range(lt): + item = data.iloc[i] + cate = item['category'] + cate2 = cate.replace(',', '_') + if cate2 not in cate2_list: + cate2_list.append(cate2) + grade = float(item['score']) + cate_list = ['rec', 'ocr', 'know', 'gen', 'spat', 'math'] + for capa in cate_list: + if capa in cate: + tot[capa] += 1 + score[capa] += grade + tot['Overall'] += 1 + tot[cate2] += 1 + score['Overall'] += grade + score[cate2] += grade + + res = defaultdict(list) + res2 = defaultdict(list) + cate_list.append('Overall') + cate2_list.append('Overall') + for k in cate_list: + res['Category'].append(k) + res['tot'].append(tot[k]) + res['acc'].append(score[k] / tot[k] * 100) + for v in cate2_list: + res2['Category'].append(v) + res2['tot'].append(tot[v]) + res2['acc'].append(score[v] / tot[v] * 100) + res = pd.DataFrame(res) + res2 = pd.DataFrame(res2) + return res, res2 + + +def MMVet_eval(eval_file, **judge_kwargs): + logger = get_logger('Evaluation') + + suffix = eval_file.split('.')[-1] + model = judge_kwargs['model'] + storage = eval_file.replace(f'.{suffix}', f'_{model}.xlsx') + tmp_file = eval_file.replace(f'.{suffix}', f'_{model}.pkl') + nproc = judge_kwargs.pop('nproc', 4) + if osp.exists(storage): + logger.warning(f'GPT scoring file {storage} already exists, will reuse it in MMVet_eval. ') + else: + data = load(eval_file) + model = build_judge(max_tokens=3, **judge_kwargs) + + lt = len(data) + lines = [data.iloc[i] for i in range(lt)] + tups = [(model, line) for line in lines] + indices = [line['index'] for line in lines] + + ans = {} + if osp.exists(tmp_file): + ans = load(tmp_file) + tups = [x for x, i in zip(tups, indices) if i not in ans] + indices = [i for i in indices if i not in ans] + + if len(indices): + new_results = track_progress_rich( + MMVet_auxeval, tups, nproc=nproc, chunksize=nproc, + keys=indices, save=tmp_file) + ans = load(tmp_file) + for k, v in zip(indices, new_results): + assert k in ans + assert ans[k]['log'] == v['log'] and ans[k]['score'] == v['score'] + + log_map, score_map = {}, {} + all_inds = [line['index'] for line in lines] + for k in all_inds: + log_map[k] = ans[k]['log'] + score_map[k] = ans[k]['score'] + data['score'] = [score_map[idx] for idx in data['index']] + data['log'] = [log_map[idx] for idx in data['index']] + dump(data, storage) + + score, score_fine = MMVet_acc(storage) + score_pth = storage.replace('.xlsx', '_score.csv') + score_fine_pth = storage.replace('.xlsx', '_score_fine.csv') + + dump(score, score_pth) + dump(score_fine, score_fine_pth) + logger.info( + f'MMVet_eval successfully finished evaluating {eval_file}, ' + f'results saved in {score_pth} and {score_fine_pth}' + ) + logger.info('Score: ') + logger.info(score) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Inference LLM Answers. ') + parser.add_argument('data', type=str, help='The question set for inference, in excel / tsv / json format. ') + parser.add_argument( + '--model', + type=str, + help='The LLM (GPT) used for inference. ', + default='gpt-4-turbo', + choices=['gpt-4-0613', 'gpt-4-turbo', 'chatgpt-1106', 'chatgpt-0613']) + parser.add_argument('--nproc', type=int, default=4) + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + MMVet_eval(eval_file=args.data, model=args.model, nproc=args.nproc, verbose=args.verbose) diff --git a/vlmeval/evaluate/multiple_choice.py b/vlmeval/evaluate/multiple_choice.py new file mode 100644 index 0000000..b806a6f --- /dev/null +++ b/vlmeval/evaluate/multiple_choice.py @@ -0,0 +1,448 @@ +import os.path as osp +import pandas as pd +from tqdm import tqdm +from vlmeval.evaluate.misc import build_judge +from vlmeval.utils import can_infer, track_progress_rich, TSVDataset +from vlmeval.smp import * +import numpy as np + +INTERNAL = os.environ.get('INTERNAL', 0) + +# abbrs = { +# 'coarse_perception': 'CP', +# 'finegrained_perception (instance-level)': 'FP-S', +# 'finegrained_perception (cross-instance)': 'FP-C', +# 'logic_reasoning': 'LR', +# 'relation_reasoning': 'RR', +# 'attribute_reasoning': 'AR' +# } + +# MMT-Bench abbrs +abbrs = { + # 'coarse_perception': 'CP', + # 'finegrained_perception (instance-level)': 'FP-S', + # 'finegrained_perception (cross-instance)': 'FP-C', + # 'logic_reasoning': 'LR', + # 'relation_reasoning': 'RR', + # 'attribute_reasoning': 'AR', + "visual_recognition": "VR", + "localization": "Loc", + "ocr": "OCR", + "counting": "Count", + "hallucination": "HLN", + "image_retrieval": "IR", + "threed": "3D", + "visual_captioning": "VC", + "visual_grounding": "VG", + "doc_understanding": "DU", + "action_recognition": "AR", + "pixel_level_perception": "PLP", + "image-to-image_translation": "I2IT", + "relation_reasoning": "RR", + "intelligence_quotient_test": "IQT", + "emotion": "Emo", + "visual_illusion": "VI", + "meme_understanding": "MemU", + "visual_prompt_understanding": "VPU", + "anomaly_detection": "AND", + "keypoint_detection": "KD", + "visual_commonsense_reasoning": "VCR", + "image_evaluation_judgement": "IEJ", + "multiple_image_analysis": "MIA", + "cross_image_matching": "CIM", + "temporal_understanding": "TU", + "visual_code": "VP", + "medical_understanding": "MedU", + "autonomous_driving": "AUD", + "discipline_knowledge_reasoning": "DKR", + "embodied_ai": "EA", + "gui_navigation": "GN" +} + + +def MMMU_preproc(data): + logger = get_logger('Evaluation') + cnt = 0 + As, Bs, Ans = list(data['A']), list(data['B']), list(data['answer']) + lt = len(data) + for i in range(lt): + if pd.isna(As[i]): + As[i] = Ans[i] + Bs[i] = 'Other Answers' + cnt += 1 + logger.info(f'During MMMU_preproc in Evaluation, {cnt} open questions are re-formulated to multi-choice ones. ') + data['A'] = As + data['B'] = Bs + return data + + +def report_acc(df): + # assert group in [None, 'category', 'l2-category'] + res = defaultdict(list) + + if 'split' in df: + splits = list(set(df['split'])) + res['split'] = splits + + else: + df['split'] = ['none'] * len(df) + res['split'] = ['none'] + + for group in [None, 'l2-category', 'category']: + if group is None: + res['Overall'] = [np.mean(df[df['split'] == sp]['hit']) for sp in res['split']] + res['Overall'].extend([np.mean(df['hit'])]) + elif group not in df: + continue + elif group == "category": + abilities = list(set(df[group])) + abilities.sort() + for ab in abilities: + ab_name = ab + sub_df = df[df[group] == ab] + res[ab_name] = [np.mean(sub_df[sub_df['split'] == sp]['hit']) for sp in res['split']] + res[ab_name].extend([np.mean(sub_df['hit'])]) + else: + abilities = list(set(df[group])) + abilities.sort() + for ab in abilities: + ab_name = abbrs[ab] if ab in abbrs else ab + sub_df = df[df[group] == ab] + res[ab_name] = [np.mean(sub_df[sub_df['split'] == sp]['hit']) for sp in res['split']] + res[ab_name].extend([np.mean(sub_df['hit'])]) + res['split'].append('ALL') + # for key, value in res.items(): + # print(key, value, len(value)) + return pd.DataFrame(res) + + +def build_prompt(question, options, prediction): + tmpl = ( + 'You are an AI assistant who will help me to match ' + 'an answer with several options of a single-choice question. ' + 'You are provided with a question, several options, and an answer, ' + 'and you need to find which option is most similar to the answer. ' + 'If the meaning of all options are significantly different from the answer, output Z. ' + 'Your should output a single uppercase character in A, B, C, D (if they are valid options), and Z. \n' + 'Example 1: \n' + 'Question: What is the main object in image?\nOptions: A. teddy bear B. rabbit C. cat D. dog\n' + 'Answer: a cute teddy bear\nYour output: A\n' + 'Example 2: \n' + 'Question: What is the main object in image?\nOptions: A. teddy bear B. rabbit C. cat D. dog\n' + 'Answer: Spider\nYour output: Z\n' + 'Example 3: \n' + 'Question: {}?\nOptions: {}\nAnswer: {}\nYour output: ' + ) + return tmpl.format(question, options, prediction) + + +def build_prompt_cn(question, options, prediction): + tmpl = ( + '你是一个帮助我匹配答案与单选题中多个选项的 AI 助手。' + '你会被提供：一个问题，多个选项，一个答案。你的任务是找到与答案意义最相近的选项。' + '如果所有选项的意义都与答案显著不同，则输出 Z。' + '你应该输出一个单个的大写字母，例如 A, B, C, D（如果它们是有效选项），或 Z。' + '例 1:' + '问题: 图中最主要的物体是什么?\n选项: A. 泰迪熊 B. 兔子 C. 猫 D. 狗\n答案: 一只可爱的泰迪熊\n输出: A\n' + '例 2: \n' + '问题: 图中最主要的物体是什么?\n选项: A. 泰迪熊 B. 兔子 C. 猫 D. 狗\n答案: 蜘蛛\n输出: Z\n' + '例 3: \n' + '问题: {}?\n选项: {}\n答案: {}\n输出: ' + ) + return tmpl.format(question, options, prediction) + + +def build_choices(item): + ret = {} + for ch in string.ascii_uppercase: + if ch in item and (not pd.isna(item[ch])): + ret[ch] = item[ch] + return ret + + +def prefetch_answer(item): + choices = build_choices(item) + return can_infer(item['prediction'], choices) + + +def extract_answer_from_item(model, item): + logger = get_logger('Evaluation') + # It will return: (pred, raw, llm_time) + choices = build_choices(item) + option_str = build_option_str(choices) + + if cn_string(item['question']): + prompt = build_prompt_cn(item['question'], option_str, item['prediction']) + else: + prompt = build_prompt(item['question'], option_str, item['prediction']) + retry = 3 + + ret = can_infer(item['prediction'], choices) + if ret: + return dict(opt=ret, log=item['prediction']) + + while retry: + ans = model.generate(prompt) + if 'Failed to obtain answer via API' in ans: + logger.warning('GPT API failed to answer. ') + else: + ret = can_infer(ans, choices) + if ret: + return dict(opt=ret, log=ans) + else: + logger.warning(f'Output includes 0 / > 1 letter among candidates {set(choices)} and Z: {ans}') + retry -= 1 + + if retry == 0: + options = list(choices) + ['Z'] if 'Z' not in choices else [] + return dict(opt=rd.choice(options), log='Failed to predict, thus randomly generate one. ') + + +def prefetch_sub_data(sub_data, answer_map, verbose=False): + lt = len(sub_data) + GT, PRED = [], [] + for i in range(lt): + item = sub_data.iloc[i] + idx = item['index'] + GT.append(answer_map[idx]) + PRED.append(prefetch_answer(item)) + if PRED[-1] and (GT[-1] != PRED[-1]): + log = ( + f'Failed in Prefetching Rolling {i}: Answer is {GT[-1]}, ' + f"Prediction is {item['prediction']}, Pre-fetched is {PRED[-1]}. " + ) + return dict(hit=0, log=log) + flag = True + for g, p in zip(GT, PRED): + if g != p: + flag = False + ret = (dict(hit=1, log='Succeed During Pre-fetching'), ) if flag else (None, ) + ret = ret + (GT, PRED) if verbose else ret + return ret if len(ret) > 1 else ret[0] + + +def eval_sub_data(model, sub_data, answer_map): + res, GT, PRED = prefetch_sub_data(sub_data, answer_map, verbose=True) + if res is not None: + return res + + lt = len(sub_data) + log = '' + for i in range(lt): + if PRED[i]: + log += f'Rolling {i} Matched.\n' + else: + res = extract_answer_from_item(model, sub_data.iloc[i]) + opt, match_log = res['opt'], res['log'] + PRED[i] = opt + if PRED[i] != GT[i]: + log += ( + f"Failed in Rolling {i}: Answer is {GT[i]}; Prediction is {sub_data.iloc[i]['prediction']}; " + f'Pre-fetched is {PRED[i]}; Match Log is {match_log}.\n' + ) + return dict(hit=0, log=log) + else: + log += ( + f"Rolling {i}: Answer is {GT[i]}, Prediction is {sub_data.iloc[i]['prediction']}, " + f'Pre-fetched is {PRED[i]}.\n' + ) + + return dict(hit=1, log=log) + + +def eval_data_groups(model, data_groups, answer_map, result, result_file, nproc=16): + prefetched = [prefetch_sub_data(g, answer_map, verbose=False) for g in data_groups] + remain = [] + for dg, pf in zip(data_groups, prefetched): + if pf: + result[dg.iloc[0]['index'] % 1e6] = pf + else: + remain.append(dg) + dump(result, result_file) + tups = [(model, x, answer_map) for x in remain] + keys = [x.iloc[0]['index'] % 1e6 for x in remain] + if len(tups) == 0: + return + + if model is None: + logger = get_logger('Evaluation') + logger.warning('Exact Matching mode, will not do GPT-based answer matching. ') + for k in keys: + result[k] = dict( + hit=0, log='Failed in Prefetch, no GPT-based answer matching under `exact_matching` policy.') + dump(result, result_file) + return + + res = track_progress_rich( + eval_sub_data, + tups, + nproc=nproc, + chunksize=nproc, + save=result_file, + keys=keys) + result = load(result_file) + for k, v in zip(keys, res): + if k in result: + assert result[k]['hit'] == v['hit'] and result[k]['log'] == v['log'] + else: + result[k] = v + dump(result, result_file) + + +def multiple_choice_eval(eval_file, dataset='default', **judge_kwargs): + logger = get_logger('Evaluation') + + # assert dataset is not None + if dataset == 'MMBench_TEST_CN': + dataset = 'MMBench_CN' + elif dataset == 'MMBench_TEST_EN': + dataset = 'MMBench' + nproc = judge_kwargs.pop('nproc', 4) + + if listinstr(['mmbench', 'ccbench'], dataset.lower()): + data = load(eval_file) + data['index'] = [int(x) for x in data['index']] + dump(data, eval_file) + + rd.seed(2680) + suffix = eval_file.split('.')[-1] + model = judge_kwargs['model'] + assert model in ['chatgpt-0613', 'exact_matching', 'gpt-4-0125'] + name_str_map = { + 'chatgpt-0613': 'openai', + 'gpt-4-0125': 'gpt4' + } + name_str = name_str_map[model] if model in name_str_map else model + + if model == 'exact_matching': + model = None + else: + if INTERNAL or gpt_key_set(): + model = build_judge(**judge_kwargs) + else: + logger.error('OPENAI_API_KEY is not set properly, will use exact matching for evaluation') + model = None + + logger.info(f'Evaluating {eval_file}') + result_file = eval_file.replace(f'.{suffix}', f'_{name_str}_result.pkl') + result = {} + if osp.exists(result_file): + result = load(result_file) + + data = load(eval_file) + data = data.sort_values(by='index') + data['prediction'] = [str(x) for x in data['prediction']] + for k in data.keys(): + data[k.lower() if k not in list(string.ascii_uppercase) else k] = data.pop(k) + + if dataset != 'default': + meta = TSVDataset(dataset).data + else: + logger.warning('Dataset is not provided, try to use the original `eval_file` as meta data. ') + meta = load(eval_file) + assert 'index' in meta and 'answer' in meta, 'Essentail columns missing in the eval_file.' + + answer_map = {i: c for i, c in zip(meta['index'], meta['answer'])} + cate_map = {i: c for i, c in zip(meta['index'], meta['category'])} if 'category' in meta else None + l2_cate_map = {i: c for i, c in zip(meta['index'], meta['l2-category'])} if 'l2-category' in meta else None + split_map = {i: c for i, c in zip(meta['index'], meta['split'])} if 'split' in meta else None + + if cate_map is not None and np.all([pd.isna(x) for x in cate_map.values()]): + cate_map = None + if l2_cate_map is not None and np.all([pd.isna(x) for x in l2_cate_map.values()]): + l2_cate_map = None + if split_map is not None and np.all([pd.isna(x) for x in split_map.values()]): + split_map = None + + if listinstr(['MMMU'], dataset): + data = MMMU_preproc(data) + answer_map = {k: (v if v in list(string.ascii_uppercase) else 'A') for k, v in answer_map.items()} + + data = data[data['index'].isin(answer_map)] + data_main = data[data['index'] < int(1e6)] + meta_idx_set = set(meta['index']) + data_main = data_main[data_main['index'].isin(meta_idx_set)] + + lt = len(data_main) + hit, tot = 0, 0 + + data_groups = [] + for i in tqdm(range(lt)): + # Dealing with the normal part + item_main = data_main.iloc[i] + idx = item_main['index'] + + if idx in result: + correct = result[idx]['hit'] + assert correct in [0, 1] + hit += correct + tot += 1 + continue + + sub_data = data[data['index'] % int(1e6) == idx] + data_groups.append(sub_data) + + if len(data_groups): + eval_data_groups( + model=model, + data_groups=data_groups, + answer_map=answer_map, + nproc=nproc, + result=result, + result_file=result_file) + + tmp_pth = f'/tmp/{timestr()}.xlsx' + dump(data_main, tmp_pth) + data_main = load(tmp_pth) + + res = load(result_file) + indices = data_main['index'] + + data_main['hit'] = [res[i]['hit'] for i in indices] + data_main['log'] = [res[i]['log'] for i in indices] + + main_idx = data_main['index'] + if cate_map is not None: + data_main['category'] = [cate_map[i] for i in main_idx] + if l2_cate_map is not None: + data_main['l2-category'] = [l2_cate_map[i] for i in main_idx] + if split_map is not None: + data_main['split'] = [split_map[i] for i in indices] + + # load split + dump(data_main, eval_file.replace(f'.{suffix}', f'_{name_str}_result.{suffix}')) + data_main = load(eval_file.replace(f'.{suffix}', f'_{name_str}_result.{suffix}')) + + acc = report_acc(data_main) + score_file = eval_file.replace(f'.{suffix}', '_acc.csv') + dump(acc, score_file) + logger.info(f'multiple_choice_eval successfully finished evaluating {eval_file}, results saved in {score_file}') + logger.info('Score: ') + logger.info(acc) + return acc + + +def parse_args(): + parser = argparse.ArgumentParser(description='Inference LLM Answers. ') + parser.add_argument('data', type=str, help='The question set for inference, in excel / tsv / json format. ') + parser.add_argument( + '--model', + type=str, + help='The LLM (GPT) used for inference. ', + default='chatgpt-0613', + choices=['chatgpt-0613', 'exact_matching', 'gpt-4-0125']) + parser.add_argument( + '--dataset', + type=str, + default='default', + help='The dataset to evaluate') + parser.add_argument('--nproc', type=int, default=6) + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + acc = multiple_choice_eval( + eval_file=args.data, model=args.model, dataset=args.dataset, nproc=args.nproc, verbose=args.verbose) diff --git a/vlmeval/evaluate/vqa_eval.py b/vlmeval/evaluate/vqa_eval.py new file mode 100644 index 0000000..33dc01d --- /dev/null +++ b/vlmeval/evaluate/vqa_eval.py @@ -0,0 +1,340 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Partly adopted from https://github.com/GT-Vision-Lab/VQA +# Copyright (c) 2014, Aishwarya Agrawal + +import re +from vlmeval.smp import * +from typing import Optional +from functools import partial + + +def _process_digit_article(inText): + outText = [] + tempText = inText.lower().split() + articles = ['a', 'an', 'the'] + manualMap = { + 'none': '0', + 'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10', + } + contractions = { + 'aint': "ain't", + 'arent': "aren't", + 'cant': "can't", + 'couldve': "could've", + 'couldnt': "couldn't", + "couldn'tve": "couldn't've", + "couldnt've": "couldn't've", + 'didnt': "didn't", + 'doesnt': "doesn't", + 'dont': "don't", + 'hadnt': "hadn't", + "hadnt've": "hadn't've", + "hadn'tve": "hadn't've", + 'hasnt': "hasn't", + 'havent': "haven't", + 'hed': "he'd", + "hed've": "he'd've", + "he'dve": "he'd've", + 'hes': "he's", + 'howd': "how'd", + 'howll': "how'll", + 'hows': "how's", + "Id've": "I'd've", + "I'dve": "I'd've", + 'Im': "I'm", + 'Ive': "I've", + 'isnt': "isn't", + 'itd': "it'd", + "itd've": "it'd've", + "it'dve": "it'd've", + 'itll': "it'll", + "let's": "let's", + 'maam': "ma'am", + 'mightnt': "mightn't", + "mightnt've": "mightn't've", + "mightn'tve": "mightn't've", + 'mightve': "might've", + 'mustnt': "mustn't", + 'mustve': "must've", + 'neednt': "needn't", + 'notve': "not've", + 'oclock': "o'clock", + 'oughtnt': "oughtn't", + "ow's'at": "'ow's'at", + "'ows'at": "'ow's'at", + "'ow'sat": "'ow's'at", + 'shant': "shan't", + "shed've": "she'd've", + "she'dve": "she'd've", + "she's": "she's", + 'shouldve': "should've", + 'shouldnt': "shouldn't", + "shouldnt've": "shouldn't've", + "shouldn'tve": "shouldn't've", + "somebody'd": 'somebodyd', + "somebodyd've": "somebody'd've", + "somebody'dve": "somebody'd've", + 'somebodyll': "somebody'll", + 'somebodys': "somebody's", + 'someoned': "someone'd", + "someoned've": "someone'd've", + "someone'dve": "someone'd've", + 'someonell': "someone'll", + 'someones': "someone's", + 'somethingd': "something'd", + "somethingd've": "something'd've", + "something'dve": "something'd've", + 'somethingll': "something'll", + 'thats': "that's", + 'thered': "there'd", + "thered've": "there'd've", + "there'dve": "there'd've", + 'therere': "there're", + 'theres': "there's", + 'theyd': "they'd", + "theyd've": "they'd've", + "they'dve": "they'd've", + 'theyll': "they'll", + 'theyre': "they're", + 'theyve': "they've", + 'twas': "'twas", + 'wasnt': "wasn't", + "wed've": "we'd've", + "we'dve": "we'd've", + 'weve': "we've", + 'werent': "weren't", + 'whatll': "what'll", + 'whatre': "what're", + 'whats': "what's", + 'whatve': "what've", + 'whens': "when's", + 'whered': "where'd", + 'wheres': "where's", + 'whereve': "where've", + 'whod': "who'd", + "whod've": "who'd've", + "who'dve": "who'd've", + 'wholl': "who'll", + 'whos': "who's", + 'whove': "who've", + 'whyll': "why'll", + 'whyre': "why're", + 'whys': "why's", + 'wont': "won't", + 'wouldve': "would've", + 'wouldnt': "wouldn't", + "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", + 'yall': "y'all", + "yall'll": "y'all'll", + "y'allll": "y'all'll", + "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", + "y'all'dve": "y'all'd've", + 'youd': "you'd", + "youd've": "you'd've", + "you'dve": "you'd've", + 'youll': "you'll", + 'youre': "you're", + 'youve': "you've", + } + for word in tempText: + word = manualMap.setdefault(word, word) + if word not in articles: + outText.append(word) + for wordId, word in enumerate(outText): + if word in contractions: + outText[wordId] = contractions[word] + outText = ' '.join(outText) + return outText + + +def hit_calculate(result, dataset_name, anls_threshold=0.5): + if listinstr(['TextVQA'], dataset_name): + return [np.mean(x['match']) for x in result] + elif listinstr(['DocVQA', 'InfoVQA'], dataset_name): + # return [1 - np.min(x['match']) >= anls_threshold for x in result] + return [0.0 if 1 - np.min(x['match']) < anls_threshold else 1 - np.min(x['match']) for x in result] + elif listinstr(['ChartQA', 'OCRVQA'], dataset_name): + return [np.max(x['match']) for x in result] + else: # default using vqa_score to calculate score + return [np.mean(x['match']) for x in result] + + +# https://github.com/google-research/pix2struct/blob/main/pix2struct/metrics.py#L81 +def relaxed_correctness(target: str, + prediction: str, + max_relative_change: float = 0.05) -> bool: + """Calculates relaxed correctness. + + The correctness tolerates certain error ratio defined by max_relative_change. + See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1: + “Following Methani et al. (2020), we use a relaxed accuracy measure for the + numeric answers to allow a minor inaccuracy that may result from the automatic + data extraction process. We consider an answer to be correct if it is within + 5% of the gold answer. For non-numeric answers, we still need an exact match + to consider an answer to be correct.” + + Args: + target: Target string. + prediction: Predicted string. + max_relative_change: Maximum relative change. + + Returns: + Whether the prediction was correct given the specified tolerance. + """ + + def _to_float(text: str) -> Optional[float]: + try: + if text.endswith('%'): + # Convert percentages to floats. + return float(text.rstrip('%')) / 100.0 + else: + return float(text) + except ValueError: + return None + prediction = str(prediction) + target = str(target) + prediction_float = _to_float(prediction) + target_float = _to_float(target) + if prediction_float is not None and target_float: + relative_change = abs(prediction_float - target_float) / abs(target_float) + return relative_change <= max_relative_change + else: + return prediction.lower() == target.lower() + + +def levenshtein_distance(s1, s2): + if len(s1) > len(s2): + s1, s2 = s2, s1 + + distances = range(len(s1) + 1) + for i2, c2 in enumerate(s2): + distances_ = [i2 + 1] + for i1, c1 in enumerate(s1): + if c1 == c2: + distances_.append(distances[i1]) + else: + distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1]))) + distances = distances_ + return distances[-1] + + +def anls_compute(groundtruth, prediction): + gt_answer = ' '.join(groundtruth.strip().lower().split()) + det_answer = ' '.join(prediction.strip().lower().split()) + dist = levenshtein_distance(gt_answer, det_answer) + length = max(len(groundtruth.upper()), len(prediction.upper())) + values = 0.0 if length == 0 else float(dist) / float(length) + return values + + +def process_answer(answer): + answer = answer.replace('\n', ' ') + answer = answer.replace('\t', ' ') + answer = answer.strip() + answer = process_punctuation(answer) + answer = _process_digit_article(answer) + return answer + + +def process_line(line, method='vqa_score'): + ret = {} + if istype(line['answer'], list): + answers = eval(line['answer']) + else: + answers = [line['answer']] + if method == 'vqa_score': + ret['gt'] = [process_answer(x) for x in answers] + ret['pred'] = process_answer(line['prediction']) + ret['match'] = [] + for current_idx, gtAnsDatum in enumerate(ret['gt']): + otherGTAns = [ + item for ret_gt_idx, item in enumerate(ret['gt']) + if ret_gt_idx != current_idx + ] + matchingAns = [ + item for item in otherGTAns if item == ret['pred'] + ] + acc = min(1, float(len(matchingAns)) / 3) + ret['match'].append(acc) + elif method == 'anls': + ret['gt'] = answers + ret['pred'] = line['prediction'] + ret['match'] = [anls_compute(x, ret['pred']) for x in ret['gt']] + elif method == 'relaxed_accuracy': + ret['gt'] = answers + ret['pred'] = line['prediction'].strip() + ret['match'] = [relaxed_correctness(ret['pred'], x) for x in ret['gt']] + elif method == 'accuracy': + ret['gt'] = answers + ret['pred'] = line['prediction'].strip() + ret['match'] = [(1.0 if (x.strip().lower() == ret['pred'].strip().lower()) else 0.0) for x in ret['gt']] + else: # default using vqa_score to calculate score + ret['gt'] = [process_answer(x) for x in answers] + ret['pred'] = process_answer(line['prediction']) + ret['match'] = [x == ret['pred'] for x in ret['gt']] + + return ret + + +def VQAEval(eval_file, dataset_name, **kwargs): + logger = get_logger('Evaluation') + data = load(eval_file) + assert 'answer' in data and 'prediction' in data + data['prediction'] = [str(x) for x in data['prediction']] + data['answer'] = [str(x) for x in data['answer']] + lt = len(data) + pool = mp.Pool(16) + lines = [data.iloc[i] for i in range(lt)] + if listinstr(['TextVQA'], dataset_name): + res = pool.map(partial(process_line, method='vqa_score'), lines) + elif listinstr(['ChartQA'], dataset_name): + res = pool.map(partial(process_line, method='relaxed_accuracy'), lines) + elif listinstr(['OCRVQA'], dataset_name): + res = pool.map(partial(process_line, method='accuracy'), lines) + elif listinstr(['DocVQA', 'InfoVQA'], dataset_name): + res = pool.map(partial(process_line, method='anls'), lines) + else: # default using vqa_score to calculate score + res = pool.map(process_line, lines) + # [np.mean(x['match']) >= full_score_weight for x in res] + hit = hit_calculate(res, dataset_name) + ret = dict() + if 'split' in data: + splits = set(data['split']) + for sp in splits: + sub = [r for l, r in zip(lines, res) if l['split'] == sp] + # [np.mean(x['match']) >= full_score_weight for x in sub] + hit = hit_calculate(sub, dataset_name) + ret[sp] = np.mean(hit) * 100 + sub = [r for l, r in zip(lines, res)] + hit = hit_calculate(sub, dataset_name) + ret['Overall'] = np.mean(hit) * 100 + else: + ret['Overall'] = np.mean(hit) * 100 + if 'category' in data: + cates = list(set(data['category'])) + cates.sort() + for c in cates: + sub = [r for l, r in zip(lines, res) if l['category'] == c] + # [np.mean(x['match']) >= full_score_weight for x in sub] + hit = hit_calculate(sub, dataset_name) + ret[c] = np.mean(hit) * 100 + ret = d2df(ret) + ret.round(2) + + suffix = eval_file.split('.')[-1] + result_file = eval_file.replace(f'.{suffix}', '_acc.csv') + logger.info(f'VQA Eval Finished. Saved to {result_file}. ') + logger.info(ret) + dump(ret, result_file) diff --git a/vlmeval/evaluate/yes_or_no.py b/vlmeval/evaluate/yes_or_no.py new file mode 100644 index 0000000..ef23cc6 --- /dev/null +++ b/vlmeval/evaluate/yes_or_no.py @@ -0,0 +1,247 @@ +from vlmeval.evaluate.misc import build_judge +from vlmeval.smp import * +from vlmeval.utils import track_progress_rich + +INTERNAL = os.environ.get('INTERNAL', 0) + + +def MME_rating(data_file): + data = load(data_file) + stats = defaultdict(dict) + lt = len(data) + for i in range(lt): + item = data.iloc[i] + category = item['category'] + image_path = item['image_path'] + score = item['score'] + if image_path not in stats[category]: + stats[category][image_path] = [] + stats[category][image_path].append(score) + + def acc(key, mode='normal'): + res = stats[key] + values = [] + for val in res.values(): + if mode == 'normal': + values.extend(val) + elif mode == 'plus': + values.append(val[0] * val[1]) + return np.mean(values) * 100 + + scores = {} + for k in stats: + scores[k] = acc(k) + acc(k, 'plus') + + super_cates = dict( + perception=[ + 'OCR', 'artwork', 'celebrity', 'color', 'count', 'existence', + 'landmark', 'position', 'posters', 'scene' + ], + reasoning=['code_reasoning', 'commonsense_reasoning', 'numerical_calculation', 'text_translation'] + ) + + ret = {} + for sc, cate_list in super_cates.items(): + base = 0 + for c in cate_list: + base += scores[c] + ret[sc] = base + ret.update(scores) + ret = d2df(ret) + return ret + + +def Hallusion_rating(data_file): + def calc_fAcc(data): + res = defaultdict(list) + lt = len(data) + for i in range(lt): + line = data.iloc[i] + res[f"{line['l2-category']}_{line['set_id']}_{line['figure_id']}"].append(line['score']) + return np.mean([np.all(x) for x in res.values()]) * 100 + + def calc_qAcc(data): + res = defaultdict(list) + lt = len(data) + for i in range(lt): + line = data.iloc[i] + res[f"{line['l2-category']}_{line['set_id']}_{line['question_id']}"].append(line['score']) + return np.mean([np.all(x) for x in res.values()]) * 100 + + def calc_aAcc(data): + return np.mean(data['score']) * 100 + + data = load(data_file) + data['set_id'] = [x.split('_')[3] for x in data['index']] + data['figure_id'] = [x.split('_')[4] for x in data['index']] + data['question_id'] = [x.split('_')[5] for x in data['index']] + + res = dict(split=[], aAcc=[], fAcc=[], qAcc=[]) + res['split'].append('Overall') + res['aAcc'].append(calc_aAcc(data)) + res['fAcc'].append(calc_fAcc(data)) + res['qAcc'].append(calc_qAcc(data)) + + if 'category' in data: + cates = list(set(data['category'])) + for c in cates: + sub = data[data['category'] == c] + res['split'].append(c) + res['aAcc'].append(calc_aAcc(sub)) + res['fAcc'].append(calc_fAcc(sub)) + res['qAcc'].append(calc_qAcc(sub)) + + if 'l2-category' in data: + cates = list(set(data['l2-category'])) + for c in cates: + sub = data[data['l2-category'] == c] + res['split'].append(c) + res['aAcc'].append(calc_aAcc(sub)) + res['fAcc'].append(calc_fAcc(sub)) + res['qAcc'].append(calc_qAcc(sub)) + ret = pd.DataFrame(res) + return ret + + +def default_rating(data_file): + data = load(data_file) + res = {} + res['Overall'] = np.mean(data['score']) * 100 + if 'category' in data: + cates = list(set(data['category'])) + cates = [c for c in cates if not pd.isna(c)] + cates.sort() + for c in cates: + sub = data[data['category'] == c] + res[c] = np.mean(sub['score']) * 100 + if 'l2-category' in data: + cates = list(set(data['l2-category'])) + cates = [c for c in cates if not pd.isna(c)] + cates.sort() + for c in cates: + sub = data[data['l2-category'] == c] + res[c] = np.mean(sub['score']) * 100 + ret = d2df(res) + return ret + + +def YOrN_match_prompt(line): + tmpl = ( + 'You are an AI assistant who will help me to match an answer with two options of a question. ' + 'The options are only Yes / No. ' + 'You are provided with a question and an answer, ' + 'and you need to find which option (Yes / No) is most similar to the answer. ' + 'If the meaning of all options are significantly different from the answer, output Unknown. ' + 'Your should output a single word among the following 3 choices: Yes, No, Unknown.\n' + 'Example 1: \n' + "Question: Is the word in this image 'Hello'?\nAnswer: The word in this image is 'Hello'.\nYour output: Yes\n" + 'Example 2: \n' + "Question: Is the word in this image 'Hello'?\n" + "Answer: The word in this image is not 'Hello'.\nYour output: No\n" + 'Example 3: \n' + 'Question: {}?\nAnswer: {}\nYour output: ' + ) + return tmpl.format(line['question'], line['prediction']) + + +def YOrN_Extraction(output): + s = output.lower() + words = process_punctuation(s).split() + if 'yes' in words and 'no' not in words: + return 'Yes' + if 'yes' not in words and 'no' in words: + return 'No' + return 'Unknown' + + +def YOrN_auxeval(model, line): + prompt = YOrN_match_prompt(line) + retry = 5 + for i in range(retry): + output = model.generate(prompt, temperature=0.5 * i) + ans = YOrN_Extraction(output) + if ans != 'Unknown': + return ans + return 'Unknown' + + +def YOrN_eval(eval_file, dataset=None, **judge_kwargs): + logger = get_logger('Evaluation') + data = load(eval_file) + data['prediction'] = [str(x) for x in data['prediction']] + storage = eval_file.replace('.xlsx', '_auxmatch.xlsx') + tmp_file = eval_file.replace('.xlsx', '_tmp.pkl') + nproc = judge_kwargs.pop('nproc', 4) + + if not osp.exists(storage): + ans_map = {k: YOrN_Extraction(v) for k, v in zip(data['index'], data['prediction'])} + if osp.exists(tmp_file): + tmp = load(tmp_file) + for k in tmp: + if ans_map[k] == 'Unknown' and tmp[k] != 'Unknown': + ans_map[k] = tmp[k] + + data['extracted'] = [ans_map[x] for x in data['index']] + unknown = data[data['extracted'] == 'Unknown'] + + if INTERNAL or gpt_key_set(): + model = build_judge(**judge_kwargs) + else: + logger.error('OPENAI_API_KEY is not set properly, will use exact matching for evaluation') + model = None + + if model is not None: + lt = len(unknown) + lines = [unknown.iloc[i] for i in range(lt)] + tups = [(model, line) for line in lines] + indices = list(unknown['index']) + if len(tups): + res = track_progress_rich( + YOrN_auxeval, tups, nproc=nproc, chunksize=nproc, keys=indices, save=tmp_file) + for k, v in zip(indices, res): + ans_map[k] = v + + data['extracted'] = [ans_map[x] for x in data['index']] + dump(data, storage) + else: + logger.warning(f'GPT matching file {storage} already exists, will reuse it in YOrN_eval. ') + + data = load(storage) + data['score'] = (data['answer'] == data['extracted']) + dump(data, storage) + + if dataset is not None and listinstr(['MME'], dataset): + score = MME_rating(storage) + elif dataset is not None and listinstr(['Hallusion'], dataset): + score = Hallusion_rating(storage) + else: + score = default_rating(storage) + + score_tgt = eval_file.replace('.xlsx', '_score.csv') + dump(score, score_tgt) + + logger.info(f'YOrN_eval successfully finished evaluating {eval_file}, results saved in {score_tgt}') + logger.info('Score: ') + logger.info(score) + return score + + +def parse_args(): + parser = argparse.ArgumentParser(description='Inference LLM Answers. ') + parser.add_argument('data', type=str, help='The question set for inference, in excel / tsv / json format. ') + parser.add_argument( + '--model', + type=str, + help='The LLM (GPT) used for inference. ', + default='chatgpt-0613', + choices=['chatgpt-0613']) + parser.add_argument('--nproc', type=int, default=4) + parser.add_argument('--dataset', type=str, default=None) + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + acc = YOrN_eval(eval_file=args.data, model=args.model, nproc=args.nproc, verbose=args.verbose, dataset=args.dataset) diff --git a/vlmeval/inference.py b/vlmeval/inference.py new file mode 100644 index 0000000..7dcf73e --- /dev/null +++ b/vlmeval/inference.py @@ -0,0 +1,183 @@ +import torch +import torch.distributed as dist +import datetime +from vlmeval.config import supported_VLM, api_models +from vlmeval.utils import TSVDataset, track_progress_rich, split_MMMU +from vlmeval.smp import * + +FAIL_MSG = 'Failed to obtain answer via API.' + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--data', type=str, nargs='+', required=True) + parser.add_argument('--model', type=str, nargs='+', required=True) + parser.add_argument('--nproc', type=int, default=4, required=True) + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + return args + + +# Only API model is accepted +def infer_data_api(work_dir, model_name, dataset_name, index_set=None, api_nproc=4, ignore_failed=False): + rank, world_size = get_rank_and_world_size() + assert rank == 0 and world_size == 1 + dataset = TSVDataset(dataset_name) + data = dataset.data + if index_set is not None: + data = data[data['index'].isin(index_set)] + + model = supported_VLM[model_name]() if isinstance(model_name, str) else model_name + assert getattr(model, 'is_api', False) + + lt, indices = len(data), list(data['index']) + structs = [dataset.build_prompt(data.iloc[i]) for i in range(lt)] + + # Corner Case + if listinstr(['MMMU'], dataset_name): + structs = [split_MMMU(s) for s in structs] + + out_file = f'{work_dir}/{model_name}_{dataset_name}_supp.pkl' + res = {} + if osp.exists(out_file): + res = load(out_file) + if ignore_failed: + res = {k: v for k, v in res.items() if FAIL_MSG not in v} + + structs = [s for i, s in zip(indices, structs) if i not in res] + indices = [i for i in indices if i not in res] + + gen_func = model.generate + # For now, we do not use split_MMMU for MMMU dataset + structs = [dict(message=struct, dataset=dataset_name) for struct in structs] + + if len(structs): + track_progress_rich(gen_func, structs, nproc=api_nproc, chunksize=api_nproc, save=out_file, keys=indices) + + res = load(out_file) + if index_set is not None: + res = {k: v for k, v in res.items() if k in index_set} + os.remove(out_file) + return res + + +def infer_data(model_name, work_dir, dataset_name, out_file, verbose=False, api_nproc=4): + prev_file = f'{work_dir}/{model_name}_{dataset_name}_PREV.pkl' + res = load(prev_file) if osp.exists(prev_file) else {} + if osp.exists(out_file): + res.update(load(out_file)) + + rank, world_size = get_rank_and_world_size() + if rank == 0: + dataset = TSVDataset(dataset_name) + if world_size > 1: + dist.barrier() + dataset = TSVDataset(dataset_name) + + sheet_indices = list(range(rank, len(dataset), world_size)) + lt = len(sheet_indices) + data = dataset.data.iloc[sheet_indices] + data_indices = [i for i in data['index']] + + # If finished, will exit without building the model + all_finished = True + for i in range(lt): + idx = data.iloc[i]['index'] + if idx not in res: + all_finished = False + if all_finished: + res = {k: res[k] for k in data_indices} + dump(res, out_file) + return + + # Data need to be inferred + data = data[~data['index'].isin(res)] + lt = len(data) + + model = supported_VLM[model_name]() if isinstance(model_name, str) else model_name + + is_api = getattr(model, 'is_api', False) + if is_api: + lt, indices = len(data), list(data['index']) + supp = infer_data_api( + work_dir=work_dir, + model_name=model_name, + dataset_name=dataset_name, + index_set=set(indices), + api_nproc=api_nproc) + for idx in indices: + assert idx in supp + res.update(supp) + res = {k: res[k] for k in data_indices} + dump(res, out_file) + return model_name + + for i in tqdm(range(lt)): + idx = data.iloc[i]['index'] + if idx in res: + continue + + if hasattr(model, 'use_custom_prompt') and model.use_custom_prompt(dataset_name): + struct = model.build_prompt(data.iloc[i], dataset=dataset_name) + else: + struct = dataset.build_prompt(data.iloc[i]) + + # Corner Case + if listinstr(['MMMU'], dataset_name): + struct = split_MMMU(struct) + + # For now, we do not use split_MMMU for MMMU dataset + response = model.generate(message=struct, dataset=dataset_name) + torch.cuda.empty_cache() + + if verbose: + print(response, flush=True) + + res[idx] = response + if (i + 1) % 20 == 0: + dump(res, out_file) + + res = {k: res[k] for k in data_indices} + dump(res, out_file) + return model + + +# A wrapper for infer_data, do the pre & post processing +def infer_data_job(model, work_dir, model_name, dataset_name, verbose=False, api_nproc=4, ignore_failed=False): + rank, world_size = get_rank_and_world_size() + result_file = osp.join(work_dir, f'{model_name}_{dataset_name}.xlsx') + + prev_file = f'{work_dir}/{model_name}_{dataset_name}_PREV.pkl' + if osp.exists(result_file): + if rank == 0: + data = load(result_file) + results = {k: v for k, v in zip(data['index'], data['prediction'])} + if not ignore_failed: + results = {k: v for k, v in results.items() if FAIL_MSG not in str(v)} + dump(results, prev_file) + if world_size > 1: + dist.barrier() + + tmpl = osp.join(work_dir, '{}' + f'{world_size}_{dataset_name}.pkl') + out_file = tmpl.format(rank) + + model = infer_data( + model, work_dir=work_dir, dataset_name=dataset_name, out_file=out_file, verbose=verbose, api_nproc=api_nproc) + if world_size > 1: + dist.barrier() + + if rank == 0: + data_all = {} + for i in range(world_size): + data_all.update(load(tmpl.format(i))) + + data = TSVDataset(dataset_name).data + for x in data['index']: + assert x in data_all + data['prediction'] = [str(data_all[x]) for x in data['index']] + data.pop('image') + + dump(data, result_file) + for i in range(world_size): + os.remove(tmpl.format(i)) + return model diff --git a/vlmeval/smp/__init__.py b/vlmeval/smp/__init__.py new file mode 100644 index 0000000..46e8968 --- /dev/null +++ b/vlmeval/smp/__init__.py @@ -0,0 +1,4 @@ +from .file import * +from .vlm import * +from .misc import * +from .log import * diff --git a/vlmeval/smp/file.py b/vlmeval/smp/file.py new file mode 100644 index 0000000..50b5446 --- /dev/null +++ b/vlmeval/smp/file.py @@ -0,0 +1,214 @@ +import json +import pickle +import pandas as pd +import os +import csv +import hashlib +import os.path as osp +import time +import numpy as np +import validators +import mimetypes + + +def LMUDataRoot(): + if 'LMUData' in os.environ and osp.exists(os.environ['LMUData']): + return os.environ['LMUData'] + home = osp.expanduser('~') + root = osp.join(home, 'LMUData') + os.makedirs(root, exist_ok=True) + return root + + +def MMBenchOfficialServer(): + root = LMUDataRoot() + for dataset in ['MMBench', 'MMBench_CN', 'MMBench_TEST_EN', 'MMBench_TEST_CN']: + if osp.exists(f'{root}/{dataset}.tsv'): + data = load(f'{root}/{dataset}.tsv') + if 'answer' in data and sum([pd.isna(x) for x in data['answer']]) == 0: + return True + return False + + +class NumpyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, + np.int16, np.int32, np.int64, np.uint8, + np.uint16, np.uint32, np.uint64)): + return int(obj) + elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): + return float(obj) + elif isinstance(obj, (np.complex_, np.complex64, np.complex128)): + return {'real': obj.real, 'imag': obj.imag} + elif isinstance(obj, (np.ndarray,)): + return obj.tolist() + elif isinstance(obj, (np.bool_)): + return bool(obj) + elif isinstance(obj, (np.void)): + return None + return json.JSONEncoder.default(self, obj) + + +# LOAD & DUMP +def dump(data, f, **kwargs): + def dump_pkl(data, pth, **kwargs): + pickle.dump(data, open(pth, 'wb')) + + def dump_json(data, pth, **kwargs): + json.dump(data, open(pth, 'w'), indent=4, ensure_ascii=False, cls=NumpyEncoder) + + def dump_jsonl(data, f, **kwargs): + lines = [json.dumps(x, ensure_ascii=False, cls=NumpyEncoder) for x in data] + with open(f, 'w', encoding='utf8') as fout: + fout.write('\n'.join(lines)) + + def dump_xlsx(data, f, **kwargs): + data.to_excel(f, index=False, engine='xlsxwriter') + + def dump_csv(data, f, quoting=csv.QUOTE_ALL): + data.to_csv(f, index=False, encoding='utf-8', quoting=quoting) + + def dump_tsv(data, f, quoting=csv.QUOTE_ALL): + data.to_csv(f, sep='\t', index=False, encoding='utf-8', quoting=quoting) + + handlers = dict(pkl=dump_pkl, json=dump_json, jsonl=dump_jsonl, xlsx=dump_xlsx, csv=dump_csv, tsv=dump_tsv) + suffix = f.split('.')[-1] + return handlers[suffix](data, f, **kwargs) + + +def load(f): + def load_pkl(pth): + return pickle.load(open(pth, 'rb')) + + def load_json(pth): + return json.load(open(pth, 'r', encoding='utf-8')) + + def load_jsonl(f): + lines = open(f, encoding='utf-8').readlines() + lines = [x.strip() for x in lines] + if lines[-1] == '': + lines = lines[:-1] + data = [json.loads(x) for x in lines] + return data + + def load_xlsx(f): + return pd.read_excel(f) + + def load_csv(f): + return pd.read_csv(f) + + def load_tsv(f): + return pd.read_csv(f, sep='\t') + + handlers = dict(pkl=load_pkl, json=load_json, jsonl=load_jsonl, xlsx=load_xlsx, csv=load_csv, tsv=load_tsv) + suffix = f.split('.')[-1] + return handlers[suffix](f) + + +def download_file(url, filename=None): + import urllib.request + from tqdm import tqdm + + class DownloadProgressBar(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + + if filename is None: + filename = url.split('/')[-1] + + with DownloadProgressBar(unit='B', unit_scale=True, + miniters=1, desc=url.split('/')[-1]) as t: + urllib.request.urlretrieve(url, filename=filename, reporthook=t.update_to) + return filename + + +def ls(dirname='.', match=[], mode='all', level=1): + if isinstance(level, str): + assert '+' in level + level = int(level[:-1]) + res = [] + for i in range(1, level + 1): + res.extend(ls(dirname, match=match, mode='file', level=i)) + return res + + if dirname == '.': + ans = os.listdir(dirname) + else: + ans = [osp.join(dirname, x) for x in os.listdir(dirname)] + assert mode in ['all', 'dir', 'file'] + assert level >= 1 and isinstance(level, int) + if level == 1: + if isinstance(match, str): + match = [match] + for m in match: + if len(m) == 0: + continue + if m[0] != '!': + ans = [x for x in ans if m in x] + else: + ans = [x for x in ans if m[1:] not in x] + if mode == 'dir': + ans = [x for x in ans if osp.isdir(x)] + elif mode == 'file': + ans = [x for x in ans if not osp.isdir(x)] + return ans + else: + dirs = [x for x in ans if osp.isdir(x)] + res = [] + for d in dirs: + res.extend(ls(d, match=match, mode=mode, level=level - 1)) + return res + + +def mrlines(fname, sp='\n'): + f = open(fname).read().split(sp) + while f != [] and f[-1] == '': + f = f[:-1] + return f + + +def mwlines(lines, fname): + with open(fname, 'w') as fout: + fout.write('\n'.join(lines)) + + +def md5(s): + hash = hashlib.new('md5') + if osp.exists(s): + with open(s, 'rb') as f: + for chunk in iter(lambda: f.read(2**20), b''): + hash.update(chunk) + else: + hash.update(s.encode('utf-8')) + return str(hash.hexdigest()) + + +def last_modified(pth): + stamp = osp.getmtime(pth) + m_ti = time.ctime(stamp) + t_obj = time.strptime(m_ti) + t = time.strftime('%Y%m%d%H%M%S', t_obj)[2:] + return t + + +def parse_file(s): + if osp.exists(s): + assert osp.isfile(s) + suffix = osp.splitext(s)[1].lower() + mime = mimetypes.types_map.get(suffix, 'unknown') + return (mime, s) + elif validators.url(s): + suffix = osp.splitext(s)[1].lower() + if suffix in mimetypes.types_map: + mime = mimetypes.types_map[suffix] + dname = osp.join(LMUDataRoot(), 'files') + os.makedirs(dname, exist_ok=True) + tgt = osp.join(dname, md5(s) + suffix) + download_file(s, tgt) + return (mime, tgt) + else: + return ('url', s) + else: + return (None, s) diff --git a/vlmeval/smp/log.py b/vlmeval/smp/log.py new file mode 100644 index 0000000..95804d5 --- /dev/null +++ b/vlmeval/smp/log.py @@ -0,0 +1,44 @@ +import logging + +logger_initialized = {} + + +def get_logger(name, log_file=None, log_level=logging.INFO, file_mode='w'): + logger = logging.getLogger(name) + if name in logger_initialized: + return logger + + for logger_name in logger_initialized: + if name.startswith(logger_name): + return logger + + stream_handler = logging.StreamHandler() + handlers = [stream_handler] + + try: + import torch.distributed as dist + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank() + else: + rank = 0 + except ImportError: + rank = 0 + + if rank == 0 and log_file is not None: + file_handler = logging.FileHandler(log_file, file_mode) + handlers.append(file_handler) + + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + for handler in handlers: + handler.setFormatter(formatter) + handler.setLevel(log_level) + logger.addHandler(handler) + + if rank == 0: + logger.setLevel(log_level) + else: + logger.setLevel(logging.ERROR) + + logger_initialized[name] = True + return logger diff --git a/vlmeval/smp/misc.py b/vlmeval/smp/misc.py new file mode 100644 index 0000000..7640fad --- /dev/null +++ b/vlmeval/smp/misc.py @@ -0,0 +1,191 @@ +# flake8: noqa: F401, F403 +import abc +import argparse +import csv +import multiprocessing as mp +import os +import os.path as osp +import copy as cp +import random as rd +import requests +import shutil +import subprocess +import warnings +import logging +import pandas as pd +from collections import OrderedDict, defaultdict +from multiprocessing import Pool, current_process +from tqdm import tqdm +import datetime +import matplotlib.pyplot as plt +import seaborn as sns +from tabulate import tabulate_formats, tabulate +from huggingface_hub import scan_cache_dir +from sty import fg, bg, ef, rs + +def process_punctuation(inText): + import re + outText = inText + punct = [ + ';', r'/', '[', ']', '"', '{', '}', '(', ')', '=', '+', '\\', '_', '-', + '>', '<', '@', '`', ',', '?', '!' + ] + commaStrip = re.compile('(\d)(,)(\d)') # noqa: W605 + periodStrip = re.compile('(?!<=\d)(\.)(?!\d)') # noqa: W605 + for p in punct: + if (p + ' ' in inText or ' ' + p in inText) or (re.search( + commaStrip, inText) is not None): + outText = outText.replace(p, '') + else: + outText = outText.replace(p, ' ') + outText = periodStrip.sub('', outText, re.UNICODE) + return outText + +def h2r(value): + if value[0] == '#': + value = value[1:] + assert len(value) == 6 + return tuple(int(value[i:i + 2], 16) for i in range(0, 6, 2)) + +def r2h(rgb): + return '#%02x%02x%02x' % rgb + +def colored(s, color): + if isinstance(color, str): + if hasattr(fg, color): + return getattr(fg, color) + s + fg.rs + color = h2r(color) + return fg(*color) + s + fg.rs + +def istype(s, type): + if isinstance(s, type): + return True + try: + return isinstance(eval(s), type) + except Exception as _: + return False + +def bincount(lst): + bins = defaultdict(lambda: 0) + for item in lst: + bins[item] += 1 + return bins + +def get_cache_path(repo_id): + hf_cache_info = scan_cache_dir() + repos = list(hf_cache_info.repos) + repo = None + for r in repos: + if r.repo_id == repo_id: + repo = r + break + if repo is None: + return None + revs = list(repo.revisions) + rev2keep, last_modified = None, 0 + for rev in revs: + if rev.last_modified > last_modified: + rev2keep, last_modified = rev, rev.last_modified + if rev2keep is None: + return None + return str(rev2keep.snapshot_path) + +def proxy_set(s): + import os + for key in ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY']: + os.environ[key] = s + +def get_rank_and_world_size(): + rank = int(os.environ.get('RANK', 0)) + world_size = int(os.environ.get('WORLD_SIZE', 1)) + return rank, world_size + +def splitlen(s, sym='/'): + return len(s.split(sym)) + +def listinstr(lst, s): + assert isinstance(lst, list) + for item in lst: + if item in s: + return True + return False + +def d2df(D): + return pd.DataFrame({x: [D[x]] for x in D}) + +def cn_string(s): + import re + if re.search(u'[\u4e00-\u9fff]', s): + return True + return False + +try: + import decord +except ImportError: + pass + +def timestr(second=True, minute=False): + s = datetime.datetime.now().strftime('%Y%m%d%H%M%S')[2:] + if second: + return s + elif minute: + return s[:-2] + else: + return s[:-4] + +def dict_merge(dct, merge_dct): + for k, _ in merge_dct.items(): + if (k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], dict)): #noqa + dict_merge(dct[k], merge_dct[k]) + else: + dct[k] = merge_dct[k] + +def youtube_dl(idx): + cmd = f'youtube-dl -f best -f mp4 "{idx}" -o {idx}.mp4' + os.system(cmd) + +def run_command(cmd): + if isinstance(cmd, str): + cmd = cmd.split() + return subprocess.check_output(cmd).decode() + +def load_env(): + logger = logging.getLogger('LOAD_ENV') + try: + import vlmeval + except ImportError: + logger.error('VLMEval is not installed. Failed to import environment variables from .env file. ') + return + pth = osp.realpath(vlmeval.__path__[0]) + pth = osp.join(pth, '../.env') + pth = osp.realpath(pth) + if not osp.exists(pth): + logger.error(f'Did not detect the .env file at {pth}, failed to load. ') + return + + from dotenv import dotenv_values + values = dotenv_values(pth) + for k, v in values.items(): + if v is not None and len(v): + os.environ[k] = v + logger.info(f'API Keys successfully loaded from {pth}') + +def pip_install_robust(package): + import sys + retry = 3 + while retry > 0: + try: + package_base = package.split('=')[0] + module = __import__(package) + return True + except ImportError: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', package]) + retry -= 1 + return False + + +def version_cmp(v1, v2, op='eq'): + from packaging import version + import operator + op_func = getattr(operator, op) + return op_func(version.parse(v1), version.parse(v2)) diff --git a/vlmeval/smp/vlm.py b/vlmeval/smp/vlm.py new file mode 100644 index 0000000..fc2db50 --- /dev/null +++ b/vlmeval/smp/vlm.py @@ -0,0 +1,133 @@ +import os +import io +import pandas as pd +import numpy as np +import string +from uuid import uuid4 +import os.path as osp +import base64 +from PIL import Image +from .file import load, dump + + +def mmqa_display(question): + question = {k.lower(): v for k, v in question.items()} + keys = list(question.keys()) + keys = [k for k in keys if k not in ['index', 'image']] + + images = question['image'] + if isinstance(images, str): + images = [images] + + idx = question.pop('index', 'XXX') + print(f'INDEX: {idx}') + + for im in images: + image = decode_base64_to_image(im, target_size=512) + display(image) # noqa: F821 + + for k in keys: + try: + if not pd.isna(question[k]): + print(f'{k.upper()}. {question[k]}') + except ValueError: + if False in pd.isna(question[k]): + print(f'{k.upper()}. {question[k]}') + + +def encode_image_to_base64(img, target_size=-1): + # if target_size == -1, will not do resizing + # else, will set the max_size ot (target_size, target_size) + if img.mode in ('RGBA', 'P'): + img = img.convert('RGB') + tmp = osp.join('/tmp', str(uuid4()) + '.jpg') + if target_size > 0: + img.thumbnail((target_size, target_size)) + img.save(tmp) + with open(tmp, 'rb') as image_file: + image_data = image_file.read() + ret = base64.b64encode(image_data).decode('utf-8') + os.remove(tmp) + return ret + + +def encode_image_file_to_base64(image_path, target_size=-1): + image = Image.open(image_path) + return encode_image_to_base64(image, target_size=target_size) + + +def decode_base64_to_image(base64_string, target_size=-1): + image_data = base64.b64decode(base64_string) + image = Image.open(io.BytesIO(image_data)) + if image.mode in ('RGBA', 'P'): + image = image.convert('RGB') + if target_size > 0: + image.thumbnail((target_size, target_size)) + return image + + +def decode_base64_to_image_file(base64_string, image_path, target_size=-1): + image = decode_base64_to_image(base64_string, target_size=target_size) + image.save(image_path) + + +def build_option_str(option_dict): + s = 'There are several options: \n' + for c, content in option_dict.items(): + if not pd.isna(content): + s += f'{c}. {content}\n' + return s + + +def isimg(s): + return osp.exists(s) or s.startswith('http') + + +def read_ok(img_path): + if not osp.exists(img_path): + return False + try: + im = Image.open(img_path) + assert im.size[0] > 0 and im.size[1] > 0 + return True + except: + return False + + +def gpt_key_set(): + openai_key = os.environ.get('OPENAI_API_KEY', None) + return isinstance(openai_key, str) and openai_key.startswith('sb-') + + +def apiok(wrapper): + s = wrapper.generate('Hello!') + return wrapper.fail_msg not in s + + +def circular_pred(df, extract_func=None): + if extract_func is None: + extract_func = lambda x: x # noqa: E731 + df = df.sort_values('index') + from vlmeval.utils import can_infer_option + shift = int(1e6) + + choices = [extract_func(x) for x in df['prediction']] + pred_map = {i: c for i, c in zip(df['index'], choices)} + flag_map = {i: True for i in pred_map if i < 1e6} + valid_map = {i: True for i in pred_map if i < 1e6} + for i in df['index']: + if i >= shift and pred_map[i] and pred_map[i - shift]: + if ( + pred_map[i] not in list(string.ascii_uppercase) or # noqa: W504 + pred_map[i - shift] not in list(string.ascii_uppercase) + ): + + valid_map[i % shift] = False + continue + if (ord(pred_map[i]) - ord(pred_map[i - shift])) % 4 == 1: + continue + else: + flag_map[i % shift] = False + flag_map = {k: v for k, v in flag_map.items() if valid_map[k]} + flags = list(flag_map.values()) + return np.mean(flags) diff --git a/vlmeval/utils/__init__.py b/vlmeval/utils/__init__.py new file mode 100644 index 0000000..9303678 --- /dev/null +++ b/vlmeval/utils/__init__.py @@ -0,0 +1,12 @@ +from .matching_util import can_infer, can_infer_option, can_infer_text +from .mp_util import track_progress_rich +from .custom_prompt import CustomPrompt +from .dataset_config import dataset_URLs, img_root_map, DATASET_TYPE, abbr2full +from .dataset import TSVDataset, split_MMMU, MMMU_result_transfer + + +__all__ = [ + 'can_infer', 'can_infer_option', 'can_infer_text', 'track_progress_rich', + 'TSVDataset', 'dataset_URLs', 'img_root_map', 'DATASET_TYPE', 'CustomPrompt', + 'split_MMMU', 'abbr2full' +] diff --git a/vlmeval/utils/custom_prompt.py b/vlmeval/utils/custom_prompt.py new file mode 100644 index 0000000..5841fbc --- /dev/null +++ b/vlmeval/utils/custom_prompt.py @@ -0,0 +1,33 @@ +from ..smp import * +from .dataset_config import img_root_map +from abc import abstractmethod + + +class CustomPrompt: + + @abstractmethod + def use_custom_prompt(self, dataset): + raise NotImplementedError + + @abstractmethod + def build_prompt(self, line, dataset): + raise NotImplementedError + + def dump_image(self, line, dataset): + ROOT = LMUDataRoot() + assert isinstance(dataset, str) + img_root = osp.join(ROOT, 'images', img_root_map[dataset] if dataset in img_root_map else dataset) + os.makedirs(img_root, exist_ok=True) + if isinstance(line['image'], list): + tgt_path = [] + assert 'image_path' in line + for img, im_name in zip(line['image'], line['image_path']): + path = osp.join(img_root, im_name) + if not read_ok(path): + decode_base64_to_image_file(img, path) + tgt_path.append(path) + else: + tgt_path = osp.join(img_root, f"{line['index']}.jpg") + if not read_ok(tgt_path): + decode_base64_to_image_file(line['image'], tgt_path) + return tgt_path diff --git a/vlmeval/utils/dataset.py b/vlmeval/utils/dataset.py new file mode 100644 index 0000000..ff843d9 --- /dev/null +++ b/vlmeval/utils/dataset.py @@ -0,0 +1,179 @@ +import pandas as pd +import hashlib +from ..smp import * +from .dataset_config import dataset_URLs, dataset_md5_dict, DATASET_TYPE +from .custom_prompt import CustomPrompt +from .matching_util import can_infer + + +def isliststr(s): + return (s[0] == '[') and (s[-1] == ']') + + +def check_md5(data_path, dataset): + if dataset not in dataset_md5_dict: + warnings.warn(f'We do not have an md5 record for dataset {dataset}, skip the md5 check. ') + return True + assert osp.exists(data_path) + with open(data_path, 'rb') as f: + hash = hashlib.new('md5') + for chunk in iter(lambda: f.read(2**20), b''): + hash.update(chunk) + if str(hash.hexdigest()) == dataset_md5_dict[dataset]: + return True + else: + warnings.warn('this data file is incomplete, so it needs to be downloaded again.') + return False + + +def split_MMMU(msgs): + text, images = None, [] + for s in msgs: + if s['type'] == 'image': + images.append(s['value']) + elif s['type'] == 'text': + assert text is None + text = s['value'] + text_segs = text.split('' + image_idx = int(seg[0]) - 1 + segs.append(dict(type='image', value=images[image_idx])) + segs.append(dict(type='text', value=seg[2:])) + return segs + + +def MMMU_result_transfer(result_path): + res = {} + result_data = load(result_path) + mcq = result_data['A'].notna() + lt = len(result_data) + for i in range(lt): + line = result_data.iloc[i] + if mcq[i]: + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + prediction = line['prediction'] + infer_prediction = can_infer(prediction, options) + res[line['id']] = infer_prediction + else: + res[line['id']] = line['prediction'] + result_json = result_path.replace('.xlsx', '.json') + dump(res, result_json) + return result_json + + +class TSVDataset(CustomPrompt): + + def __init__(self, dataset='MMBench', skip_noimg=True): + + self.data_root = LMUDataRoot() + assert osp.exists(self.data_root) + + self.dataset = dataset + self.dataset_type = DATASET_TYPE(dataset) + + if dataset in dataset_URLs: + url = dataset_URLs[dataset] + file_name = url.split('/')[-1] + data_path = osp.join(self.data_root, file_name) + + if osp.exists(data_path) and check_md5(data_path, dataset): + pass + else: + # print(osp.exists(data_path), check_md5(data_path, dataset)) + warnings.warn('The dataset tsv is not downloaded') + download_file(url, data_path) + else: + data_path = osp.join(self.data_root, dataset + '.tsv') + assert osp.exists(data_path) + + data = load(data_path) + self.skip_noimg = skip_noimg + if skip_noimg: + data = data[~pd.isna(data['image'])] + + # Prompt for Captioning + if listinstr(['COCO'], dataset): + data['question'] = [( + 'Please describe this image in general. Directly provide the description, ' + 'do not include prefix like "This image depicts". ' + )] * len(data) + + data['index'] = [str(x) for x in data['index']] + data['image'] = [str(x) for x in data['image']] + + image_map = {x: y for x, y in zip(data['index'], data['image'])} + for k in image_map: + if len(image_map[k]) <= 64: + idx = image_map[k] + assert idx in image_map and len(image_map[idx]) > 64 + image_map[k] = image_map[idx] + + data['image'] = [ + eval(image_map[k]) if isliststr(image_map[k]) else image_map[k] + for k in data['index'] + ] + if 'image_path' in data: + data['image_path'] = [ + eval(pths) if isliststr(pths) else pths for pths in data['image_path'] + ] + if np.all([istype(x, int) for x in data['index']]): + data['index'] = [int(x) for x in data['index']] + + self.data = data + + def __len__(self): + return len(self.data) + + def build_prompt(self, line, dataset=None): + if dataset is None: + dataset = self.dataset + + if isinstance(line, int): + line = self.data.iloc[line] + + tgt_path = self.dump_image(line, dataset) + + prompt = line['question'] + if DATASET_TYPE(dataset) == 'multi-choice': + question = line['question'] + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + options_prompt = 'Options:\n' + for key, item in options.items(): + options_prompt += f'{key}. {item}\n' + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + prompt = '' + if hint is not None: + prompt += f'Hint: {hint}\n' + prompt += f'Question: {question}\n' + if len(options): + prompt += options_prompt + prompt += 'Please select the correct answer from the options above. \n' + elif DATASET_TYPE(dataset) == 'VQA': + if listinstr(['ocrvqa', 'textvqa', 'chartqa', 'docvqa'], dataset.lower()): + prompt += '\nPlease try to answer the question with short words or phrases if possible\n.' + + msgs = [] + if isinstance(tgt_path, list): + msgs.extend([dict(type='image', value=p) for p in tgt_path]) + else: + msgs = [dict(type='image', value=tgt_path)] + msgs.append(dict(type='text', value=prompt)) + + return msgs + + def display(self, line): + if isinstance(line, int): + line = self.data.iloc[line] + mmqa_display(line) diff --git a/vlmeval/utils/dataset_config.py b/vlmeval/utils/dataset_config.py new file mode 100644 index 0000000..215cf43 --- /dev/null +++ b/vlmeval/utils/dataset_config.py @@ -0,0 +1,137 @@ +from ..smp import listinstr + +dataset_URLs = { + 'MMBench_DEV_EN': 'https://opencompass.openxlab.space/utils/VLMEval/MMBench_DEV_EN.tsv', + 'MMBench_TEST_EN': 'https://opencompass.openxlab.space/utils/VLMEval/MMBench_TEST_EN.tsv', + 'MMBench_DEV_CN': 'https://opencompass.openxlab.space/utils/VLMEval/MMBench_DEV_CN.tsv', + 'MMBench_TEST_CN': 'https://opencompass.openxlab.space/utils/VLMEval/MMBench_TEST_CN.tsv', + 'MMBench': 'https://opencompass.openxlab.space/utils/VLMEval/MMBench.tsv', # Link Invalid, Internal Only + 'MMBench_CN': 'https://opencompass.openxlab.space/utils/VLMEval/MMBench_CN.tsv', # Link Invalid, Internal Only + 'CCBench': 'https://opencompass.openxlab.space/utils/VLMEval/CCBench.tsv', + 'MME': 'https://opencompass.openxlab.space/utils/VLMEval/MME.tsv', + 'SEEDBench_IMG': 'https://opencompass.openxlab.space/utils/VLMEval/SEEDBench_IMG.tsv', + 'CORE_MM': 'https://opencompass.openxlab.space/utils/VLMEval/CORE_MM.tsv', + 'MMVet': 'https://opencompass.openxlab.space/utils/VLMEval/MMVet.tsv', + 'COCO_VAL': 'https://opencompass.openxlab.space/utils/VLMEval/COCO_VAL.tsv', + 'OCRVQA_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/OCRVQA_TEST.tsv', + 'OCRVQA_TESTCORE': 'https://opencompass.openxlab.space/utils/VLMEval/OCRVQA_TESTCORE.tsv', + 'TextVQA_VAL': 'https://opencompass.openxlab.space/utils/VLMEval/TextVQA_VAL.tsv', + 'MMMU_DEV_VAL': 'https://opencompass.openxlab.space/utils/VLMEval/MMMU_DEV_VAL.tsv', + 'MMMU_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/MMMU_TEST.tsv', + 'MathVista_MINI': 'https://opencompass.openxlab.space/utils/VLMEval/MathVista_MINI.tsv', + 'ScienceQA_VAL': 'https://opencompass.openxlab.space/utils/VLMEval/ScienceQA_VAL.tsv', + 'ScienceQA_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/ScienceQA_TEST.tsv', + 'HallusionBench': 'https://opencompass.openxlab.space/utils/VLMEval/HallusionBench.tsv', + 'DocVQA_VAL': 'https://opencompass.openxlab.space/utils/VLMEval/DocVQA_VAL.tsv', + 'DocVQA_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/DocVQA_TEST.tsv', + 'InfoVQA_VAL': 'https://opencompass.openxlab.space/utils/VLMEval/InfoVQA_VAL.tsv', + 'InfoVQA_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/InfoVQA_TEST.tsv', + 'AI2D_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/AI2D_TEST.tsv', + 'LLaVABench': 'https://opencompass.openxlab.space/utils/VLMEval/LLaVABench.tsv', + 'OCRBench': 'https://opencompass.openxlab.space/utils/VLMEval/OCRBench.tsv', + 'ChartQA_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/ChartQA_TEST.tsv', + 'MMStar': 'https://opencompass.openxlab.space/utils/VLMEval/MMStar.tsv', + 'RealWorldQA': 'https://opencompass.openxlab.space/utils/VLMEval/RealWorldQA.tsv', + 'MMT-Bench_VAL': 'LMUData/MMT-Bench_VAL.tsv', + 'MMT-Bench_TEST': 'LMUData/MMT-Bench_TEST.tsv', + 'MMT-Bench_ALL': 'LMUData/MMT-Bench_ALL.tsv' +} + +dataset_md5_dict = { + 'MMBench_DEV_EN': 'b6caf1133a01c6bb705cf753bb527ed8', + 'MMBench_TEST_EN': '6939fadb0ce626fefc0bdc9c64efc528', + 'MMBench_DEV_CN': '08b8fc3324a5ed74155350f57be69fbd', + 'MMBench_TEST_CN': '7e1239baf0ee4c8b513e19705a0f317e', + 'MMBench': '4115aea3383f3dd0083be6a633e0f820', # Link Invalid, Internal Only + 'MMBench_CN': '2e053ffc90ea598b1feae13c36dc13ee', # Link Invalid, Internal Only + 'CCBench': '1de88b4257e7eee3f60b18d45eda6f07', + 'MME': 'b36b43c3f09801f5d368627fb92187c3', + 'SEEDBench_IMG': '68017231464752261a2526d6ca3a10c0', + 'CORE_MM': '8a8da2f2232e79caf98415bfdf0a202d', + 'MMVet': '748aa6d4aa9d4de798306a63718455e3', + 'COCO_VAL': '72a5079dead060269ac222c5aa5128af', + 'OCRVQA_TEST': 'ca46a6d74b403e9d6c0b670f6fc00db9', + 'OCRVQA_TESTCORE': 'c5239fe77db8bdc1f2ad8e55e0d1fe97', + 'TextVQA_VAL': 'b233b31f551bbf4056f2f955da3a92cd', + 'MMMU_DEV_VAL': '521afc0f3bf341e6654327792781644d', + 'MMMU_TEST': 'c19875d11a2d348d07e5eb4bdf33166d', + 'MathVista_MINI': 'f199b98e178e5a2a20e7048f5dcb0464', + 'ScienceQA_VAL': '96320d05e142e585e7204e72affd29f3', + 'ScienceQA_TEST': 'e42e9e00f9c59a80d8a5db35bc32b71f', + 'HallusionBench': '0c23ac0dc9ef46832d7a24504f2a0c7c', + 'DocVQA_VAL': 'd5ee77e1926ff10690d469c56b73eabf', + 'DocVQA_TEST': '6a2f28cac26ef2d3447374e8c6f6c8e9', + 'InfoVQA_VAL': '2342e9c225222f0ef4dec545ebb126fe', + 'InfoVQA_TEST': 'df535bf51b88dc9718252c34131a6227', + 'AI2D_TEST': '0f593e0d1c7df9a3d69bf1f947e71975', + 'LLaVABench': 'd382a093f749a697820d3dadd61c8428', + 'OCRBench': 'e953d98a987cc6e26ef717b61260b778', + 'ChartQA_TEST': 'c902e0aa9be5582a7aad6dcf52734b42', + 'MMStar': 'e1ecd2140806c1b1bbf54b43372efb9e', + 'RealWorldQA': 'b7fc0bee84d301627cf37581ef139035', + 'MMT-Bench_VAL': '337effe5a0c9e9372b7fc1ce1c4cb385', + 'MMT-Bench_TEST': 'cc5ee58196ef4284d67c92cf35029bfd', + 'MMT-Bench_ALL': '8cbb9a6a3febdd72efb2c16ea5f19022' +} + +img_root_map = {k: k for k in dataset_URLs} +img_root_map.update({ + 'MMBench_DEV_EN': 'MMBench', + 'MMBench_TEST_EN': 'MMBench', + 'MMBench_DEV_CN': 'MMBench', + 'MMBench_TEST_CN': 'MMBench', + 'MMBench_CN': 'MMBench', # Link Invalid, Internal Only + 'COCO_VAL': 'COCO', + 'OCRVQA_TEST': 'OCRVQA', + 'OCRVQA_TESTCORE': 'OCRVQA', + 'TextVQA_VAL': 'TextVQA', + 'MMMU_DEV_VAL': 'MMMU', + 'MMMU_TEST': 'MMMU', + 'MathVista_MINI': 'MathVista', + 'HallusionBench': 'Hallusion', + 'DocVQA_VAL': 'DocVQA', + 'DocVQA_TEST': 'DocVQA_TEST', + 'OCRBench': 'OCRBench', + 'ChartQA_TEST': 'ChartQA_TEST', + 'InfoVQA_VAL': 'InfoVQA_VAL', + 'InfoVQA_TEST': 'InfoVQA_TEST', + 'MMStar': 'MMStar', + 'RealWorldQA': 'RealWorldQA', + 'MMT-Bench_VAL': 'MMT-Bench_VAL', + 'MMT-Bench_TEST': 'MMT-Bench_TEST', + 'MMT-Bench_ALL': 'MMT-Bench_ALL', +}) + +assert set(dataset_URLs) == set(img_root_map) + + +def DATASET_TYPE(dataset): + # Dealing with Custom Dataset + dataset = dataset.lower() + if listinstr(['mmbench', 'seedbench', 'ccbench', 'mmmu', 'scienceqa', 'ai2d', 'mmstar', 'realworldqa', 'mmt-bench_val', 'mmt-bench_test', 'mmt-bench_all'], dataset): + return 'multi-choice' + elif listinstr(['mme', 'hallusion'], dataset): + return 'Y/N' + elif 'coco' in dataset: + return 'Caption' + elif listinstr(['ocrvqa', 'textvqa', 'chartqa', 'mathvista', 'docvqa', 'infovqa', 'llavabench', + 'mmvet', 'ocrbench'], dataset): + return 'VQA' + else: + if dataset not in dataset_URLs: + import warnings + warnings.warn(f"Dataset {dataset} not found in dataset_URLs, will use 'multi-choice' as the default TYPE.") + return 'multi-choice' + else: + return 'QA' + + +def abbr2full(s): + datasets = [x for x in img_root_map] + ins = [s in d for d in datasets] + if sum(ins) == 1: + for d in datasets: + if s in d: + return d + else: + return s diff --git a/vlmeval/utils/matching_util.py b/vlmeval/utils/matching_util.py new file mode 100644 index 0000000..ccdf1b7 --- /dev/null +++ b/vlmeval/utils/matching_util.py @@ -0,0 +1,69 @@ +import string +import copy as cp +import os +from ..smp import * + + +def can_infer_option(answer, choices): + verbose = os.environ.get('VERBOSE', 0) + # Choices is a dictionary + if 'Failed to obtain answer via API' in answer: + return False + + reject_to_answer = [ + "Sorry, I can't help with images of people yet.", + "I can't process this file.", + "I'm sorry, but without the image provided", + 'Cannot determine the answer' + ] + for err in reject_to_answer: + if err in answer: + return 'Z' + + def count_choice(splits, choices, prefix='', suffix=''): + cnt = 0 + for c in choices: + if prefix + c + suffix in splits: + cnt += 1 + return cnt + + answer_mod = cp.copy(answer) + chars = '.()[],:;!*#{}' + for c in chars: + answer_mod = answer_mod.replace(c, ' ') + + splits = [x.strip() for x in answer_mod.split()] + count = count_choice(splits, choices) + + if count == 1: + for ch in choices: + if 'A' in splits and len(splits) > 3 and verbose: + logger = get_logger('Evaluation') + logger.info(f'A might be a quantifier in the string: {answer}.') + return False + if ch in splits: + return ch + elif count == 0 and count_choice(splits, {'Z', ''}) == 1: + return 'Z' + return False + + +def can_infer_text(answer, choices): + answer = answer.lower() + assert isinstance(choices, dict) + for k in choices: + assert k in string.ascii_uppercase + choices[k] = str(choices[k]).lower() + cands = [] + for k in choices: + if choices[k] in answer: + cands.append(k) + if len(cands) == 1: + return cands[0] + return False + + +def can_infer(answer, choices): + answer = str(answer) + copt = can_infer_option(answer, choices) + return copt if copt else can_infer_text(answer, choices) diff --git a/vlmeval/utils/mp_util.py b/vlmeval/utils/mp_util.py new file mode 100644 index 0000000..2322e2d --- /dev/null +++ b/vlmeval/utils/mp_util.py @@ -0,0 +1,191 @@ +from multiprocessing import Pool +import os +from typing import Callable, Iterable, Sized + +from rich.progress import (BarColumn, MofNCompleteColumn, Progress, Task, + TaskProgressColumn, TextColumn, TimeRemainingColumn) +from rich.text import Text +import os.path as osp +import portalocker +from ..smp import load, dump + + +class _Worker: + """Function wrapper for ``track_progress_rich``""" + + def __init__(self, func) -> None: + self.func = func + + def __call__(self, inputs): + inputs, idx = inputs + if not isinstance(inputs, (tuple, list, dict)): + inputs = (inputs, ) + + if isinstance(inputs, dict): + return self.func(**inputs), idx + else: + return self.func(*inputs), idx + + +class _SkipFirstTimeRemainingColumn(TimeRemainingColumn): + """Skip calculating remaining time for the first few times. + + Args: + skip_times (int): The number of times to skip. Defaults to 0. + """ + + def __init__(self, *args, skip_times=0, **kwargs): + super().__init__(*args, **kwargs) + self.skip_times = skip_times + + def render(self, task: Task) -> Text: + """Show time remaining.""" + if task.completed <= self.skip_times: + return Text('-:--:--', style='progress.remaining') + return super().render(task) + + +def _tasks_with_index(tasks): + """Add index to tasks.""" + for idx, task in enumerate(tasks): + yield task, idx + + +def track_progress_rich(func: Callable, + tasks: Iterable = tuple(), + task_num: int = None, + nproc: int = 1, + chunksize: int = 1, + description: str = 'Processing', + save=None, keys=None, + color: str = 'blue') -> list: + """Track the progress of parallel task execution with a progress bar. The + built-in :mod:`multiprocessing` module is used for process pools and tasks + are done with :func:`Pool.map` or :func:`Pool.imap_unordered`. + + Args: + func (callable): The function to be applied to each task. + tasks (Iterable or Sized): A tuple of tasks. There are several cases + for different format tasks: + - When ``func`` accepts no arguments: tasks should be an empty + tuple, and ``task_num`` must be specified. + - When ``func`` accepts only one argument: tasks should be a tuple + containing the argument. + - When ``func`` accepts multiple arguments: tasks should be a + tuple, with each element representing a set of arguments. + If an element is a ``dict``, it will be parsed as a set of + keyword-only arguments. + Defaults to an empty tuple. + task_num (int, optional): If ``tasks`` is an iterator which does not + have length, the number of tasks can be provided by ``task_num``. + Defaults to None. + nproc (int): Process (worker) number, if nuproc is 1, + use single process. Defaults to 1. + chunksize (int): Refer to :class:`multiprocessing.Pool` for details. + Defaults to 1. + description (str): The description of progress bar. + Defaults to "Process". + color (str): The color of progress bar. Defaults to "blue". + + Examples: + >>> import time + + >>> def func(x): + ... time.sleep(1) + ... return x**2 + >>> track_progress_rich(func, range(10), nproc=2) + + Returns: + list: The task results. + """ + if save is not None: + assert osp.exists(osp.dirname(save)) or osp.dirname(save) == '' + if not osp.exists(save): + dump({}, save) + if keys is not None: + assert len(keys) == len(tasks) + + if not callable(func): + raise TypeError('func must be a callable object') + if not isinstance(tasks, Iterable): + raise TypeError( + f'tasks must be an iterable object, but got {type(tasks)}') + if isinstance(tasks, Sized): + if len(tasks) == 0: + if task_num is None: + raise ValueError('If tasks is an empty iterable, ' + 'task_num must be set') + else: + tasks = tuple(tuple() for _ in range(task_num)) + else: + if task_num is not None and task_num != len(tasks): + raise ValueError('task_num does not match the length of tasks') + task_num = len(tasks) + + if nproc <= 0: + raise ValueError('nproc must be a positive number') + + skip_times = nproc * chunksize if nproc > 1 else 0 + prog_bar = Progress( + TextColumn('{task.description}'), + BarColumn(), + _SkipFirstTimeRemainingColumn(skip_times=skip_times), + MofNCompleteColumn(), + TaskProgressColumn(show_speed=True), + ) + + worker = _Worker(func) + task_id = prog_bar.add_task( + total=task_num, color=color, description=description) + tasks = _tasks_with_index(tasks) + + # Use single process when nproc is 1, else use multiprocess. + with prog_bar: + if nproc == 1: + results = [] + for task in tasks: + result, idx = worker(task) + results.append(worker(task)[0]) + if save is not None: + with portalocker.Lock(save, timeout=5) as fh: + ans = load(save) + ans[keys[idx]] = result + + if os.environ.get('VERBOSE', True): + print(keys[idx], result, flush=True) + + dump(ans, save) + fh.flush() + os.fsync(fh.fileno()) + + prog_bar.update(task_id, advance=1, refresh=True) + else: + with Pool(nproc) as pool: + results = [] + unordered_results = [] + gen = pool.imap_unordered(worker, tasks, chunksize) + try: + for result in gen: + result, idx = result + unordered_results.append((result, idx)) + + if save is not None: + with portalocker.Lock(save, timeout=5) as fh: + ans = load(save) + ans[keys[idx]] = result + + if os.environ.get('VERBOSE', False): + print(keys[idx], result, flush=True) + + dump(ans, save) + fh.flush() + os.fsync(fh.fileno()) + + results.append(None) + prog_bar.update(task_id, advance=1, refresh=True) + except Exception as e: + prog_bar.stop() + raise e + for result, idx in unordered_results: + results[idx] = result + return results diff --git a/vlmeval/vlm/__init__.py b/vlmeval/vlm/__init__.py new file mode 100644 index 0000000..8d2492c --- /dev/null +++ b/vlmeval/vlm/__init__.py @@ -0,0 +1,25 @@ +import torch + +torch.set_grad_enabled(False) +torch.manual_seed(1234) +from .base import BaseModel +from .cogvlm import CogVlm +from .emu import Emu +from .idefics import IDEFICS, IDEFICS2 +from .instructblip import InstructBLIP +from .llava import LLaVA, LLaVA_Next, LLaVA_XTuner +from .minicpm_v import MiniCPM_V +from .minigpt4 import MiniGPT4 +from .mmalaya import MMAlaya +from .monkey import Monkey, MonkeyChat +from .mplug_owl2 import mPLUG_Owl2 +from .omnilmm import OmniLMM12B +from .open_flamingo import OpenFlamingo +from .pandagpt import PandaGPT +from .qwen_vl import QwenVL, QwenVLChat +from .transcore_m import TransCoreM +from .visualglm import VisualGLM +from .xcomposer import ShareCaptioner, XComposer, XComposer2, XComposer2_4KHD +from .yi_vl import Yi_VL +from .internvl_chat import InternVLChat +from .deepseek_vl import DeepSeekVL diff --git a/vlmeval/vlm/base.py b/vlmeval/vlm/base.py new file mode 100644 index 0000000..5d9ed0a --- /dev/null +++ b/vlmeval/vlm/base.py @@ -0,0 +1,153 @@ +from ..smp import * +from ..utils.dataset_config import img_root_map +from abc import abstractmethod + + +class BaseModel: + + INTERLEAVE = False + allowed_types = ['text', 'image'] + + def use_custom_prompt(self, dataset): + """Whether to use custom prompt for the given dataset. + + Args: + dataset (str): The name of the dataset. + + Returns: + bool: Whether to use custom prompt. If True, will call `build_prompt` of the VLM to build the prompt. + Default to False. + """ + return False + + @abstractmethod + def build_prompt(self, line, dataset): + """Build custom prompts for a specific dataset. Called only if `use_custom_prompt` returns True. + + Args: + line (line of pd.DataFrame): The raw input line. + dataset (str): The name of the dataset. + + Returns: + str: The built message. + """ + raise NotImplementedError + + def dump_image(self, line, dataset): + """Dump the image(s) of the input line to the corresponding dataset folder. + + Args: + line (line of pd.DataFrame): The raw input line. + dataset (str): The name of the dataset. + + Returns: + str | list[str]: The paths of the dumped images. + """ + ROOT = LMUDataRoot() + assert isinstance(dataset, str) + img_root = osp.join(ROOT, 'images', img_root_map[dataset] if dataset in img_root_map else dataset) + os.makedirs(img_root, exist_ok=True) + if isinstance(line['image'], list): + tgt_path = [] + assert 'image_path' in line + for img, im_name in zip(line['image'], line['image_path']): + path = osp.join(img_root, im_name) + if not read_ok(path): + decode_base64_to_image_file(img, path) + tgt_path.append(path) + else: + tgt_path = osp.join(img_root, f"{line['index']}.jpg") + if not read_ok(tgt_path): + decode_base64_to_image_file(line['image'], tgt_path) + tgt_path = [tgt_path] + return tgt_path + + @abstractmethod + def generate_inner(self, message, dataset=None): + raise NotImplementedError + + def check_content(self, msgs): + """Check the content type of the input. Four types are allowed: str, dict, liststr, listdict. + """ + if isinstance(msgs, str): + return 'str' + if isinstance(msgs, dict): + return 'dict' + if isinstance(msgs, list): + types = [self.check_content(m) for m in msgs] + if all(t == 'str' for t in types): + return 'liststr' + if all(t == 'dict' for t in types): + return 'listdict' + return 'unknown' + + def preproc_content(self, inputs): + """Convert the raw input messages to a list of dicts. + + Args: + inputs: raw input messages. + + Returns: + list(dict): The preprocessed input messages. Will return None if failed to preprocess the input. + """ + if self.check_content(inputs) == 'str': + return [dict(type='text', value=inputs)] + elif self.check_content(inputs) == 'dict': + assert 'type' in inputs and 'value' in inputs + return [inputs] + elif self.check_content(inputs) == 'liststr': + res = [] + for s in inputs: + mime, pth = parse_file(s) + if mime is None or mime == 'unknown': + res.append(dict(type='text', value=s)) + else: + res.append(dict(type=mime.split('/')[0], value=pth)) + return res + elif self.check_content(inputs) == 'listdict': + for item in inputs: + assert 'type' in item and 'value' in item + mime, s = parse_file(item['value']) + if mime is None: + assert item['type'] == 'text' + else: + assert mime.split('/')[0] == item['type'] + item['value'] = s + return inputs + else: + return None + + def generate(self, message, dataset=None): + """Generate the output message. + + Args: + message (list[dict]): The input message. + dataset (str, optional): The name of the dataset. Defaults to None. + + Returns: + str: The generated message. + """ + assert self.check_content(message) in ['str', 'dict', 'liststr', 'listdict'], f'Invalid input type: {message}' + message = self.preproc_content(message) + assert message is not None and self.check_content(message) == 'listdict' + for item in message: + assert item['type'] in self.allowed_types, f'Invalid input type: {item["type"]}' + return self.generate_inner(message, dataset) + + def message_to_promptimg(self, message): + assert not self.INTERLEAVE + model_name = self.__class__.__name__ + warnings.warn( + f'Model {model_name} does not support interleaved input. ' + 'Will use the first image and aggregated texts as prompt. ') + num_images = len([x for x in message if x['type'] == 'image']) + if num_images == 0: + prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text']) + image = None + elif num_images == 1: + prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text']) + image = [x['value'] for x in message if x['type'] == 'image'][0] + else: + prompt = '\n'.join([x['value'] if x['type'] == 'text' else '' for x in message]) + image = [x['value'] for x in message if x['type'] == 'image'][0] + return prompt, image diff --git a/vlmeval/vlm/cogvlm.py b/vlmeval/vlm/cogvlm.py new file mode 100644 index 0000000..5999c9c --- /dev/null +++ b/vlmeval/vlm/cogvlm.py @@ -0,0 +1,84 @@ +import torch +from PIL import Image +from .base import BaseModel +from ..smp import * +from ..utils import DATASET_TYPE +from transformers import AutoModelForCausalLM, LlamaTokenizer + + +class CogVlm(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, + name='cogvlm-chat', + tokenizer_name='lmsys/vicuna-7b-v1.5', + **kwargs): + self.tokenizer = LlamaTokenizer.from_pretrained(tokenizer_name) + self.kwargs = kwargs + self.model = AutoModelForCausalLM.from_pretrained( + f'THUDM/{name}-hf', + torch_dtype=torch.bfloat16, + trust_remote_code=True, + ).to('cuda').eval() + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + if dataset is not None and DATASET_TYPE(dataset) == 'multi-choice': + question = line['question'] + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + option_candidate = string.ascii_uppercase + options = { + cand: line[cand] + for cand in option_candidate + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + prompt = question + + if not cn_string(prompt): + prompt = prompt + '\n' + "Answer with the option's letter from the given choices directly." + else: + prompt = prompt + '\n' + '请直接回答选项字母。' + else: + prompt = line['question'] + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=p) for p in tgt_path]) + + return message + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + + image = Image.open(image_path).convert('RGB') + inputs = self.model.build_conversation_input_ids( + self.tokenizer, query=prompt, history=[], images=[image]) # chat mode + inputs = { + 'input_ids': inputs['input_ids'].unsqueeze(0).to('cuda'), + 'token_type_ids': inputs['token_type_ids'].unsqueeze(0).to('cuda'), + 'attention_mask': inputs['attention_mask'].unsqueeze(0).to('cuda'), + 'images': [[inputs['images'][0].to('cuda').to(torch.bfloat16)]], + } + gen_kwargs = {'max_length': 2048, 'do_sample': False} + gen_kwargs.update(self.kwargs) + + with torch.no_grad(): + outputs = self.model.generate(**inputs, **gen_kwargs) + outputs = outputs[:, inputs['input_ids'].shape[1]:] + response = self.tokenizer.decode(outputs[0]) + response = response.split('')[0].strip() + return response diff --git a/vlmeval/vlm/deepseek_vl.py b/vlmeval/vlm/deepseek_vl.py new file mode 100644 index 0000000..492e7d4 --- /dev/null +++ b/vlmeval/vlm/deepseek_vl.py @@ -0,0 +1,69 @@ +import sys +import torch +from transformers import AutoModelForCausalLM +import warnings +from .base import BaseModel + + +class DeepSeekVL(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = True + + def check_install(self): + try: + import deepseek_vl + except ImportError: + warnings.warn( + 'Please first install deepseek_vl from source codes in: https://github.com/deepseek-ai/DeepSeek-VL') + sys.exit(-1) + + def __init__(self, model_path='deepseek-ai/deepseek-vl-1.3b-chat', **kwargs): + self.check_install() + assert model_path is not None + self.model_path = model_path + from deepseek_vl.models import VLChatProcessor + + self.vl_chat_processor = VLChatProcessor.from_pretrained(model_path) + self.tokenizer = self.vl_chat_processor.tokenizer + + model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True) + self.model = model.to(torch.bfloat16).cuda().eval() + + torch.cuda.empty_cache() + default_kwargs = dict(max_new_tokens=512, do_sample=False, use_cache=True) + default_kwargs.update(kwargs) + self.kwargs = default_kwargs + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def prepare_inputs(self, message): + content, images = '', [] + for s in message: + if s['type'] == 'image': + images.append(s['value']) + content += '' + elif s['type'] == 'text': + content += s['value'] + conversation = [ + dict(role='User', content=content, images=images), + dict(role='Assistant', content='') + ] + return conversation + + def generate_inner(self, message, dataset=None): + conversation = self.prepare_inputs(message) + from deepseek_vl.utils.io import load_pil_images + pil_images = load_pil_images(conversation) + prepare_inputs = self.vl_chat_processor(conversations=conversation, images=pil_images, force_batchify=True) + prepare_inputs = prepare_inputs.to(self.model.device) + inputs_embeds = self.model.prepare_inputs_embeds(**prepare_inputs) + + outputs = self.model.language_model.generate( + inputs_embeds=inputs_embeds, + attention_mask=prepare_inputs.attention_mask, + pad_token_id=self.tokenizer.eos_token_id, + bos_token_id=self.tokenizer.bos_token_id, + eos_token_id=self.tokenizer.eos_token_id, + **self.kwargs) + answer = self.tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True) + return answer diff --git a/vlmeval/vlm/emu.py b/vlmeval/vlm/emu.py new file mode 100644 index 0000000..1051c79 --- /dev/null +++ b/vlmeval/vlm/emu.py @@ -0,0 +1,89 @@ +import os +import torch +from PIL import Image +import os.path as osp +from .base import BaseModel +from ..smp import * + + +class Emu(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = True + + def __init__(self, + model_path='BAAI/Emu2-Chat', + **kwargs): + + self.model_path = model_path + assert osp.exists(model_path) or splitlen(model_path) == 2 + + from transformers import AutoModelForCausalLM, AutoTokenizer + from accelerate import init_empty_weights, infer_auto_device_map, dispatch_model + + local_rank = os.environ.get('LOCAL_RANK', 0) + + device_num = torch.cuda.device_count() + assert local_rank * 2 <= device_num, 'The number of devices does not match the world size' + assert device_num >= 2, 'You need at least 2 GPUs to use EMU' + + device_1 = local_rank + device_2 = local_rank + device_num // 2 + + torch.cuda.set_device(device_1) + torch.cuda.set_device(device_2) + + tokenizer = AutoTokenizer.from_pretrained(model_path) # "BAAI/Emu2-Chat" + self.tokenizer = tokenizer + with init_empty_weights(): + model = AutoModelForCausalLM.from_pretrained( + model_path, # "BAAI/Emu2-Chat" + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=True, + trust_remote_code=True) + + device_map = infer_auto_device_map( + model, + max_memory={ + device_1: '38GiB', + device_2: '38GiB' + }, + no_split_module_classes=['Block', 'LlamaDecoderLayer']) + + # input and output logits should be on same device + device_map['model.decoder.lm.lm_head'] = device_1 + + model = dispatch_model( + model, + device_map=device_map).eval() + + self.model = model + kwargs_default = dict(max_new_tokens=512, length_penalty=-1) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def generate_inner(self, message, dataset=None): + query, images = '', [] + for item in message: + if item['type'] == 'image': + images.append(Image.open(item['value']).convert('RGB')) + query += '[]' + elif item['type'] == 'text': + query += item['value'] + + inputs = self.model.build_input_ids( + text=[query], + tokenizer=self.tokenizer, + image=images + ) + + with torch.no_grad(): + outputs = self.model.generate( + input_ids=inputs['input_ids'], + attention_mask=inputs['attention_mask'], + image=inputs['image'].to(torch.bfloat16), + **self.kwargs) + + output_text = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + return output_text[0] diff --git a/vlmeval/vlm/idefics.py b/vlmeval/vlm/idefics.py new file mode 100644 index 0000000..8f1517b --- /dev/null +++ b/vlmeval/vlm/idefics.py @@ -0,0 +1,80 @@ +import torch +from PIL import Image +import os.path as osp +import warnings +from .base import BaseModel +from ..smp import splitlen, listinstr +from transformers import AutoProcessor, AutoModelForVision2Seq +from transformers.image_utils import load_image + + +class IDEFICS(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = True + + def __init__(self, + model_pth='HuggingFaceM4/idefics-9b-instruct', + **kwargs): + assert osp.exists(model_pth) or splitlen(model_pth) == 2 + from transformers import IdeficsForVisionText2Text, AutoProcessor + self.model = IdeficsForVisionText2Text.from_pretrained( + model_pth, torch_dtype=torch.bfloat16, device_map='auto') + self.processor = AutoProcessor.from_pretrained(model_pth) + kwargs_default = {'max_length': 512} + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + self.file_root = osp.dirname(__file__) + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def generate_inner(self, message, dataset=None): + prompts = ['Users:'] + [x['value'] for x in message] + ['', '\nAssistant: '] + inputs = self.processor(prompts, add_end_of_utterance_token=False, return_tensors='pt').to('cuda') + exit_condition = self.processor.tokenizer('', add_special_tokens=False).input_ids + bad_words_ids = self.processor.tokenizer( + ['', ''], + add_special_tokens=False).input_ids + + generated_ids = self.model.generate( + **inputs, eos_token_id=exit_condition, bad_words_ids=bad_words_ids, **self.kwargs) + generated_text = self.processor.batch_decode(generated_ids, skip_special_tokens=True) + text = generated_text[0].split('\nAssistant: ')[-1] + return text + + +class IDEFICS2(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = True + + def __init__(self, + model_path='HuggingFaceM4/idefics2-8b', + **kwargs): + assert model_path is not None + self.model_path = model_path + # set size={"longest_edge": 448, "shortest_edge": 378} for limited maximum image resolution, such as MMMU. + self.processor = AutoProcessor.from_pretrained(model_path, size={'longest_edge': 448, 'shortest_edge': 378}) + self.model = AutoModelForVision2Seq.from_pretrained(model_path, device_map='cuda') + kwargs_default = {'max_new_tokens': 512} + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + torch.cuda.empty_cache() + + def generate_inner(self, message, dataset=None): + prompt, images = '', [] + for msg in message: + if msg['type'] == 'image': + img = load_image(msg['value']) + images.append(img) + prompt += '' + elif msg['type'] == 'text': + prompt += msg['value'] + prompt = f'User:{prompt}\nAssistant:' + inputs = self.processor(text=prompt, images=images, return_tensors='pt') + inputs = {k: v.to(self.model.device) for k, v in inputs.items()} + + generated_ids = self.model.generate(**inputs, **self.kwargs) + generated_text = self.processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + response = generated_text.split('\nAssistant:')[-1].strip() + return response diff --git a/vlmeval/vlm/instructblip.py b/vlmeval/vlm/instructblip.py new file mode 100644 index 0000000..6c7c3e9 --- /dev/null +++ b/vlmeval/vlm/instructblip.py @@ -0,0 +1,57 @@ +import torch +from PIL import Image +import os.path as osp +import sys +from .base import BaseModel +from ..smp import * + + +class InstructBLIP(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def __init__(self, name): + self.config_map = { + 'instructblip_7b': 'misc/blip2_instruct_vicuna7b.yaml', + 'instructblip_13b': 'misc/blip2_instruct_vicuna13b.yaml', + } + + self.file_path = __file__ + config_root = osp.dirname(self.file_path) + + try: + from lavis.models import load_preprocess + from omegaconf import OmegaConf + from lavis.common.registry import registry + except: + warnings.warn('Please install lavis before using InstructBLIP. ') + sys.exit(-1) + + assert name in self.config_map + cfg_path = osp.join(config_root, self.config_map[name]) + cfg = OmegaConf.load(cfg_path) + + model_cfg = cfg.model + assert osp.exists(model_cfg.llm_model) or splitlen(model_cfg.llm_model) == 2 + model_cls = registry.get_model_class(name='blip2_vicuna_instruct') + model = model_cls.from_config(model_cfg) + model.eval() + + self.device = torch.device('cuda') if torch.cuda.is_available() else 'cpu' + device = self.device + model.to(device) + self.model = model + self.kwargs = {'max_length': 512} + + preprocess_cfg = cfg.preprocess + vis_processors, _ = load_preprocess(preprocess_cfg) + self.vis_processors = vis_processors + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + vis_processors = self.vis_processors + raw_image = Image.open(image_path).convert('RGB') + image_tensor = vis_processors['eval'](raw_image).unsqueeze(0).to(self.device) + outputs = self.model.generate(dict(image=image_tensor, prompt=prompt)) + return outputs[0] diff --git a/vlmeval/vlm/internvl_chat.py b/vlmeval/vlm/internvl_chat.py new file mode 100644 index 0000000..6a7d4df --- /dev/null +++ b/vlmeval/vlm/internvl_chat.py @@ -0,0 +1,213 @@ +import torch +from transformers import AutoTokenizer, AutoModel, CLIPImageProcessor +import warnings +from PIL import Image +from .base import BaseModel +from ..smp import * +from ..utils import DATASET_TYPE +import pandas as pd +import string +import torchvision.transforms as T + +from torchvision.transforms.functional import InterpolationMode + + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + + +def build_transform(input_size): + MEAN, STD = IMAGENET_MEAN, IMAGENET_STD + transform = T.Compose([ + T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img), + T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=MEAN, std=STD) + ]) + return transform + + +def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size): + best_ratio_diff = float('inf') + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_aspect_ratio = ratio[0] / ratio[1] + ratio_diff = abs(aspect_ratio - target_aspect_ratio) + if ratio_diff < best_ratio_diff: + best_ratio_diff = ratio_diff + best_ratio = ratio + elif ratio_diff == best_ratio_diff: + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: + best_ratio = ratio + return best_ratio + + +def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False): + orig_width, orig_height = image.size + aspect_ratio = orig_width / orig_height + + # calculate the existing image aspect ratio + target_ratios = set( + (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if + i * j <= max_num and i * j >= min_num) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, target_ratios, orig_width, orig_height, image_size) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ( + (i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size + ) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images + + +def load_image(image_file, input_size=448, max_num=6): + image = Image.open(image_file).convert('RGB') + transform = build_transform(input_size=input_size) + images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num) + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + return pixel_values + + +class InternVLChat(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='OpenGVLab/InternVL-Chat-Chinese-V1-1', **kwargs): + assert model_path is not None + self.model_path = model_path + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=False) + device = torch.cuda.current_device() + self.device = device + self.model = AutoModel.from_pretrained(model_path, torch_dtype=torch.bfloat16, + trust_remote_code=True).eval().to(device) + self.image_size = self.model.config.vision_config.image_size + + if 'V1-1' in model_path: + kwargs_default = dict(do_sample=False, max_new_tokens=1024, top_p=None, num_beams=5) + else: + kwargs_default = dict(do_sample=False, max_new_tokens=1024, top_p=None, num_beams=1) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def use_custom_prompt(self, dataset): + return True + + def build_multi_choice_prompt(self, line, dataset=None): + question = line['question'] + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + prompt = question + + if len(options): + prompt += '\n请直接回答选项字母。' if cn_string( + prompt) else "\nAnswer with the option's letter from the given choices directly." + else: + prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.' + + return prompt + + def build_prompt(self, line, dataset=None): + assert self.use_custom_prompt(dataset) + assert dataset is None or isinstance(dataset, str) + tgt_path = self.dump_image(line, dataset) + + if 'V1-1' in self.model_path: + kwargs_default = dict(do_sample=False, max_new_tokens=1024, top_p=None, num_beams=5) + else: + kwargs_default = dict(do_sample=False, max_new_tokens=1024, top_p=None, num_beams=1) + self.kwargs = kwargs_default + if dataset is not None and listinstr(['MME'], dataset): + question = line['question'] + prompt = question + ' Answer the question using a single word or phrase.' + if 'V1-2' not in self.model_path: + self.kwargs = dict(do_sample=True, max_new_tokens=5, top_k=50, num_beams=5, top_p=0.9) + elif dataset is not None and listinstr(['HallusionBench'], dataset): + question = line['question'] + prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.' + elif dataset is not None and DATASET_TYPE(dataset) == 'multi-choice': + prompt = self.build_multi_choice_prompt(line, dataset) + elif dataset is not None and DATASET_TYPE(dataset) == 'VQA': + if 'MathVista' in dataset: + prompt = line['question'] + elif listinstr(['LLaVABench'], dataset): + question = line['question'] + prompt = question + '\nAnswer this question in detail.' + elif listinstr(['MMVet'], dataset): + prompt = line['question'] + else: + question = line['question'] + prompt = question + '\nAnswer the question using a single word or phrase.' + else: + prompt = line['question'] + + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + + return message + + def generate_v1_5(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + if dataset is not None and listinstr(['DocVQA_TEST', 'DocVQA_VAL', 'ChartQA_TEST'], dataset): + self.max_num = 12 + elif dataset is not None and listinstr(['InfoVQA_VAL', 'InfoVQA_TEST', 'OCRBench'], dataset): + self.max_num = 24 + else: + self.max_num = 6 + pixel_values = load_image(image_path, max_num=self.max_num).cuda().to(torch.bfloat16) + with torch.no_grad(): + response = self.model.chat(self.tokenizer, pixel_values=pixel_values, + question=prompt, generation_config=self.kwargs) + response = response.split('[UNUSED_TOKEN_145]')[0] + return response + + def generate_v1_2(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + image = Image.open(image_path).convert('RGB') + image = image.resize((self.image_size, self.image_size)) + image_processor = CLIPImageProcessor.from_pretrained(self.model_path) + pixel_values = image_processor(images=image, return_tensors='pt').pixel_values + pixel_values = pixel_values.to(torch.bfloat16).to(self.device) + with torch.no_grad(): + response = self.model.chat(self.tokenizer, pixel_values=pixel_values, + question=prompt, generation_config=self.kwargs) + return response + + def generate_inner(self, message, dataset=None): + if 'V1-5' in self.model_path: + return self.generate_v1_5(message, dataset) + else: + return self.generate_v1_2(message, dataset) diff --git a/vlmeval/vlm/llava/__init__.py b/vlmeval/vlm/llava/__init__.py new file mode 100644 index 0000000..edf40bb --- /dev/null +++ b/vlmeval/vlm/llava/__init__.py @@ -0,0 +1,4 @@ +from .llava import LLaVA, LLaVA_Next +from .llava_xtuner import LLaVA_XTuner + +__all__ = ['LLaVA', 'LLaVA_Next', 'LLaVA_XTuner'] diff --git a/vlmeval/vlm/llava/llava.py b/vlmeval/vlm/llava/llava.py new file mode 100644 index 0000000..958aad0 --- /dev/null +++ b/vlmeval/vlm/llava/llava.py @@ -0,0 +1,260 @@ +import torch +from PIL import Image +from abc import abstractproperty +import sys +import os.path as osp +from ..base import BaseModel +from ...smp import * +from ...utils import DATASET_TYPE + + +class LLaVA(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = True + + def __init__(self, + model_pth='liuhaotian/llava_v1.5_7b', + **kwargs): + try: + from llava.model.builder import load_pretrained_model + from llava.mm_utils import get_model_name_from_path + except: + warnings.warn('Please install llava before using LLaVA') + sys.exit(-1) + + warnings.warn('Please install the latest version of llava from github before you evaluate the LLaVA model. ') + assert osp.exists(model_pth) or splitlen(model_pth) == 2 + + if model_pth == 'Lin-Chen/ShareGPT4V-7B': + model_name = 'llava-v1.5-7b' + elif model_pth == 'Lin-Chen/ShareGPT4V-13B': + model_name = 'llava-v1.5-13b' + else: + model_name = get_model_name_from_path(model_pth) + + try: + self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model( + model_path=model_pth, + model_base=None, + model_name=model_name, + device='cpu', + device_map='cpu' + ) + except: + if 'ShareGPT4V' in model_pth: + import llava + warnings.warn( + 'Please manually remove the encoder type check in ' + f'{llava.__path__[0]}/model/multimodal_encoder/builder.py ' + 'Line 8 to use the ShareGPT4V model. ') + else: + warnings.warn('Unknown error when loading LLaVA model.') + exit(-1) + + self.model = self.model.cuda() + self.conv_mode = 'llava_v1' + + kwargs_default = dict(do_sample=False, temperature=0, max_new_tokens=512, top_p=None, num_beams=1) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert self.use_custom_prompt(dataset) + assert dataset is None or isinstance(dataset, str) + tgt_path = self.dump_image(line, dataset) + + question = line['question'] + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + prompt = question + + if len(options): + prompt += ( + '\n请直接回答选项字母。' if cn_string(prompt) else + "\nAnswer with the option's letter from the given choices directly." + ) + else: + prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.' + + message = [dict(type='image', value=s) for s in tgt_path] + message.append(dict(type='text', value=prompt)) + return message + + def generate_inner(self, message, dataset=None): + from llava.mm_utils import process_images, tokenizer_image_token, KeywordsStoppingCriteria + from llava.constants import ( + IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN) + from llava.conversation import conv_templates, SeparatorStyle + + # Support interleave text and image + conv = conv_templates[self.conv_mode].copy() + conv.append_message(conv.roles[0], 'PLACEHOLDER') + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + content, images = '', [] + for msg in message: + if msg['type'] == 'text': + content += msg['value'] + elif msg['type'] == 'image': + if self.model.config.mm_use_im_start_end: + content += DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + else: + content += DEFAULT_IMAGE_TOKEN + '\n' + images.append(msg['value']) + + images = [Image.open(s).convert('RGB') for s in images] + args = abstractproperty() + args.image_aspect_ratio = 'pad' + image_tensor = process_images(images, self.image_processor, args).to('cuda', dtype=torch.float16) + prompt = prompt.replace('PLACEHOLDER', content) + + input_ids = tokenizer_image_token( + prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda() + stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 + keywords = [stop_str] + stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids) + with torch.inference_mode(): + output_ids = self.model.generate( + input_ids, images=image_tensor, stopping_criteria=[stopping_criteria], **self.kwargs) + + output = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() + return output + + +class LLaVA_Next(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = True + + def __init__(self, model_pth='llava-hf/llava-v1.6-vicuna-7b-hf', **kwargs): + import transformers + assert version_cmp(transformers.__version__, '4.39.0', 'ge') + from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration + self.model_pth = model_pth + if '34b' in model_pth.lower(): + self.processor = LlavaNextProcessor.from_pretrained(self.model_pth, use_fast=False) + else: + self.processor = LlavaNextProcessor.from_pretrained(self.model_pth) + flash_attn_flag = False + try: + import flash_attn + flash_attn_flag = True + except ImportError: + pass + + if flash_attn_flag: + model = LlavaNextForConditionalGeneration.from_pretrained( + self.model_pth, torch_dtype=torch.float16, low_cpu_mem_usage=True, use_flash_attention_2=True) + else: + model = LlavaNextForConditionalGeneration.from_pretrained( + self.model_pth, torch_dtype=torch.float16, low_cpu_mem_usage=True) + + model = model.eval() + self.model = model.cuda() + kwargs_default = dict(do_sample=False, temperature=0, max_new_tokens=512, top_p=None, num_beams=1) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def apply_prompt_template(self, message): + model_pth = self.model_pth.lower() + if 'mistral' in model_pth: + template = '[INST] PLACEHOLDER [/INST]' + elif 'vicuna' in model_pth: + template = ( + 'A chat between a curious human and an artificial intelligence assistant. ' + "The assistant gives helpful, detailed, and polite answers to the human's questions. " + 'USER: PLACEHOLDER ASSISTANT:' + ) + elif '34b' in model_pth: + template = ( + '<|im_start|>system\nAnswer the questions.<|im_end|><|im_start|>user\nPLACEHOLDER<|im_end|>' + '<|im_start|>assistant\n' + ) + else: + raise NotImplementedError(f'Prompt template for {model_pth} not implemented.') + + prompt = template.replace('PLACEHOLDER', f'\n{prompt}') + return prompt + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert self.use_custom_prompt(dataset) + assert dataset is None or isinstance(dataset, str) + tgt_path = self.dump_image(line, dataset) + + question = line['question'] + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + prompt = question + + if len(options): + prompt += ( + '\n请直接回答选项字母。' if cn_string(prompt) else + "\nAnswer with the option's letter from the given choices directly." + ) + else: + prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.' + message = [dict(type='image', value=s) for s in tgt_path] + message.append(dict(type='text', value=prompt)) + return message + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + prompt = prompt.replace('', '[ImageHere]') + if prompt.find('[ImageHere]') != prompt.rfind('[ImageHere]'): + prompt += '\nThere exists multiple images in the conversation, but only the first one is displayed.' + + image = Image.open(image_path).convert('RGB') + prompt = self.apply_prompt_template(prompt) + + inputs = self.processor(prompt, image, return_tensors='pt').to('cuda') + output = self.model.generate(**inputs, **self.kwargs) + answer = self.processor.decode(output[0], skip_special_token=True) + if '' in answer: + answer = answer.replace('', '').strip() + if '[/INST]' in answer: + answer = answer.split('[/INST]')[1].strip() + elif 'ASSISTANT:' in answer: + answer = answer.split('ASSISTANT:')[1].strip() + elif 'assistant\n' in answer: + answer = answer.split('assistant\n')[1].strip() + + if '' in answer: + answer = answer.split('')[0].strip() + if '<|im_end|>' in answer: + answer = answer.split('<|im_end|>')[0].strip() + + return answer diff --git a/vlmeval/vlm/llava/llava_xtuner.py b/vlmeval/vlm/llava/llava_xtuner.py new file mode 100644 index 0000000..4f41bbb --- /dev/null +++ b/vlmeval/vlm/llava/llava_xtuner.py @@ -0,0 +1,226 @@ +import os +import os.path as osp +import string +import sys +import warnings + +import pandas as pd +import torch +from huggingface_hub import snapshot_download +from PIL import Image +from transformers import (AutoModel, AutoModelForCausalLM, AutoTokenizer, + CLIPImageProcessor, CLIPVisionModel, + GenerationConfig, StoppingCriteriaList) + +from ..base import BaseModel +from ...smp import cn_string, get_cache_path +from ...utils import DATASET_TYPE + + +class LLaVA_XTuner(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def __init__(self, + llava_path, + llm_path=None, + visual_encoder_path='openai/clip-vit-large-patch14-336', + visual_select_layer=-2, + prompt_template=None, + stop_words=[], + torch_dtype=torch.float16): + try: + from peft import PeftModel + from xtuner.utils import PROMPT_TEMPLATE, StopWordStoppingCriteria + except Exception: + warnings.warn( + 'Please install xtuner with `pip install -U xtuner` before ' + 'using LLaVA_XTuner') + sys.exit(-1) + + if not osp.isdir(llava_path): + cache_path = get_cache_path(llava_path) + if cache_path is not None: + llava_path = cache_path + else: + llava_path = snapshot_download(repo_id=llava_path) + assert osp.exists(llava_path) and osp.isdir(llava_path) + + # build visual_encoder + if 'llm' in os.listdir(llava_path): + assert llm_path is None, ( + "Please don't specify the `llm_path` since passed " + '`llava_path` contains a LLM!') + llm_path = osp.join(llava_path, 'llm') + else: + assert llm_path is not None, 'Please specify the `llm_path`!' + + llm = AutoModelForCausalLM.from_pretrained(llm_path, + trust_remote_code=True, + torch_dtype=torch_dtype, + device_map='cpu') + tokenizer = AutoTokenizer.from_pretrained(llm_path, + trust_remote_code=True, + encode_special_tokens=True) + print(f'Load LLM from {llm_path}') + + # build visual_encoder + if 'visual_encoder' in os.listdir(llava_path): + assert visual_encoder_path is None, ( + "Please don't specify the `visual_encoder_path` since passed " + '`llava_path` contains a visual encoder!') + visual_encoder_path = osp.join(llava_path, 'visual_encoder') + else: + assert visual_encoder_path is not None, ( + 'Please specify the `visual_encoder_path`!') + visual_encoder = CLIPVisionModel.from_pretrained( + visual_encoder_path, torch_dtype=torch_dtype, device_map='cpu') + image_processor = CLIPImageProcessor.from_pretrained( + visual_encoder_path) + print(f'Load visual_encoder from {visual_encoder_path}') + + # load adapter + if 'llm_adapter' in os.listdir(llava_path): + adapter_path = osp.join(llava_path, 'llm_adapter') + llm = PeftModel.from_pretrained(llm, + adapter_path, + trust_remote_code=True, + device_map='cpu') + print(f'Load LLM adapter from {llava_path}') + if 'visual_encoder_adapter' in os.listdir(llava_path): + adapter_path = osp.join(llava_path, 'visual_encoder_adapter') + visual_encoder = PeftModel.from_pretrained(visual_encoder, + adapter_path, + trust_remote_code=True, + device_map='cpu') + print(f'Load visual_encoder adapter from {llava_path}') + + # build projector + projector_path = osp.join(llava_path, 'projector') + projector = AutoModel.from_pretrained(projector_path, + trust_remote_code=True, + torch_dtype=torch_dtype, + device_map='cpu') + print(f'Load projector from {llava_path}') + + llm.eval() + visual_encoder.eval() + projector.eval() + + self.llm = llm.cuda() + self.tokenizer = tokenizer + self.visual_encoder = visual_encoder.cuda() + self.image_processor = image_processor + self.projector = projector.cuda() + self.visual_select_layer = visual_select_layer + if prompt_template is not None: + self.prompt_template = PROMPT_TEMPLATE[prompt_template] + stop_words += self.prompt_template.get('STOP_WORDS', []) + else: + self.prompt_template = None + + self.stop_criteria = StoppingCriteriaList() + for word in stop_words: + self.stop_criteria.append( + StopWordStoppingCriteria(self.tokenizer, word)) + + def build_gen_config(self, dataset): + gen_kwargs = dict(max_new_tokens=512, + do_sample=True, + temperature=1, + num_beams=5, + eos_token_id=self.tokenizer.eos_token_id, + pad_token_id=self.tokenizer.pad_token_id + if self.tokenizer.pad_token_id is not None else + self.tokenizer.eos_token_id) + # For single word generation + if (dataset is not None + and DATASET_TYPE(dataset) in ['multi-choice', 'Y/N']): + gen_kwargs.update( + dict(max_new_tokens=5, do_sample=False, num_beams=1)) + return GenerationConfig(**gen_kwargs) + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert self.use_custom_prompt(dataset) + assert dataset is None or isinstance(dataset, str) + tgt_path = self.dump_image(line, dataset) + + question = line['question'] + hint = line['hint'] if ('hint' in line + and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + + if not cn_string(question): + prompt = question + '\n' + ("Answer with the option's letter " + 'from the given choices directly.') + else: + prompt = question + '\n' + '请直接回答选项字母。' + + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + return message + + def generate_inner(self, message, dataset=None): + from xtuner.dataset.utils import expand2square + from xtuner.model.utils import prepare_inputs_labels_for_multimodal + from xtuner.utils import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX + prompt, image_path = self.message_to_promptimg(message) + image = Image.open(image_path).convert('RGB') + image = expand2square( + image, + tuple(int(x * 255) for x in self.image_processor.image_mean)) + image = self.image_processor.preprocess( + image, return_tensors='pt')['pixel_values'][0] + image = image.cuda().unsqueeze(0) + visual_outputs = self.visual_encoder(image, output_hidden_states=True) + pixel_values = self.projector( + visual_outputs.hidden_states[self.visual_select_layer][:, 1:]) + + inputs = DEFAULT_IMAGE_TOKEN + '\n' + prompt + + if self.prompt_template: + inputs = self.prompt_template['INSTRUCTION'].format(input=inputs) + + chunk_encode = [] + for idx, chunk in enumerate(inputs.split(DEFAULT_IMAGE_TOKEN)): + if idx == 0: + cur_encode = self.tokenizer(chunk) + else: + cur_encode = self.tokenizer(chunk, add_special_tokens=False) + chunk_encode.append(cur_encode) + assert len(chunk_encode) == 2 + ids = [] + for idx, cur_chunk_encode in enumerate(chunk_encode): + ids.extend(cur_chunk_encode['input_ids']) + if idx != len(chunk_encode) - 1: + ids.append(IMAGE_TOKEN_INDEX) + ids = torch.tensor(ids).cuda().unsqueeze(0) + mm_inputs = prepare_inputs_labels_for_multimodal( + llm=self.llm, input_ids=ids, pixel_values=pixel_values) + + gen_config = self.build_gen_config(dataset) + generate_output = self.llm.generate( + **mm_inputs, + generation_config=gen_config, + streamer=None, + bos_token_id=self.tokenizer.bos_token_id, + stopping_criteria=self.stop_criteria) + predict = self.tokenizer.decode(generate_output[0], + skip_special_tokens=True).strip() + return predict diff --git a/vlmeval/vlm/minicpm_v.py b/vlmeval/vlm/minicpm_v.py new file mode 100644 index 0000000..7605482 --- /dev/null +++ b/vlmeval/vlm/minicpm_v.py @@ -0,0 +1,85 @@ +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +from .base import BaseModel +from ..smp import * +from ..utils import DATASET_TYPE + + +class MiniCPM_V(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='openbmb/MiniCPM-V', **kwargs): + assert model_path is not None + self.model_path = model_path + print(f'load from {self.model_path}') + self.model = AutoModel.from_pretrained(self.model_path, trust_remote_code=True) + self.model = self.model.to(dtype=torch.bfloat16) + self.model.eval().cuda() + self.kwargs = kwargs + self.tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True) + torch.cuda.empty_cache() + self.num_beams = 1 if self.model_path == 'openbmb/MiniCPM-V' else 3 + + def use_custom_prompt(self, dataset): + assert dataset is not None + if listinstr(['MMMU'], dataset): + return True + return False + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + question = line['question'] + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + options_prompt = 'Options:\n' + for key, item in options.items(): + options_prompt += f'{key}. {item}\n' + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + prompt = '' + if hint is not None: + prompt += f'Hint: {hint}\n' + prompt += f'{question}\n' + if len(options): + prompt += options_prompt + prompt = 'Study the image carefully and pick the option associated with the correct answer. \ + Focus solely on selecting the option and avoid including any other content.\n' + prompt + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=p) for p in tgt_path]) + + return message + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + image = Image.open(image_path).convert('RGB') + msgs = [{'role': 'user', 'content': prompt}] + if DATASET_TYPE(dataset) == 'multi-choice': + max_new_tokens = 20 + elif DATASET_TYPE(dataset) == 'Y/N': + max_new_tokens = 100 + else: + max_new_tokens = 1024 + + default_kwargs = dict( + max_new_tokens=max_new_tokens, + sampling=False, + num_beams=self.num_beams + ) + default_kwargs.update(self.kwargs) + res, _, _ = self.model.chat( + image=image, + msgs=msgs, + context=None, + tokenizer=self.tokenizer, + **default_kwargs + ) + return res diff --git a/vlmeval/vlm/minigpt4.py b/vlmeval/vlm/minigpt4.py new file mode 100644 index 0000000..382317f --- /dev/null +++ b/vlmeval/vlm/minigpt4.py @@ -0,0 +1,85 @@ +import torch +import sys +import os.path as osp +import warnings +from transformers import StoppingCriteriaList +from .base import BaseModel + + +class MiniGPT4(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def __init__(self, + mode='v2', + root='/mnt/petrelfs/share_data/duanhaodong/MiniGPT-4/', + temperature=1, + max_out_len=512): + + if root is None: + warnings.warn( + 'Please set root to the directory of MiniGPT-4, which is cloned from here: ' + 'https://github.com/Vision-CAIR/MiniGPT-4. ' + ) + + if mode == 'v2': + cfg = 'minigptv2_eval.yaml' + elif mode == 'v1_7b': + cfg = 'minigpt4_7b_eval.yaml' + elif mode == 'v1_13b': + cfg = 'minigpt4_13b_eval.yaml' + else: + raise NotImplementedError + + self.mode = mode + self.temperature = temperature + self.max_out_len = max_out_len + self.root = root + this_dir = osp.dirname(__file__) + + self.cfg = osp.join(this_dir, 'misc', cfg) + sys.path.append(self.root) + + from omegaconf import OmegaConf + from minigpt4.common.registry import registry + from minigpt4.conversation.conversation import StoppingCriteriaSub, CONV_VISION_Vicuna0, CONV_VISION_minigptv2 + + device = torch.cuda.current_device() + self.device = device + + cfg_path = self.cfg + cfg = OmegaConf.load(cfg_path) + + model_cfg = cfg.model + model_cfg.device_8bit = device + model_cls = registry.get_model_class(model_cfg.arch) + model = model_cls.from_config(model_cfg) + model = model.to(device) + model.eval() + vis_processor_cfg = cfg.datasets.cc_sbu_align.vis_processor.train + vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg) + self.model = model + self.vis_processor = vis_processor + + self.CONV_VISION = CONV_VISION_minigptv2 if self.mode == 'v2' else CONV_VISION_Vicuna0 + stop_words_ids = [[835], [2277, 29937]] + stop_words_ids = [torch.tensor(ids).to(device) for ids in stop_words_ids] + self.stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)]) + + def generate_inner(self, message, dataset=None): + from minigpt4.conversation.conversation import Chat + prompt, image_path = self.message_to_promptimg(message) + if self.mode == 'v2': + chat = Chat(self.model, self.vis_processor, device=self.device) + else: + chat = Chat(self.model, self.vis_processor, device=self.device, stopping_criteria=self.stopping_criteria) + + chat_state = self.CONV_VISION.copy() + img_list = [] + _ = chat.upload_img(image_path, chat_state, img_list) + chat.encode_img(img_list) + chat.ask(prompt, chat_state) + with torch.inference_mode(): + msg = chat.answer(conv=chat_state, img_list=img_list)[0] + return msg diff --git a/vlmeval/vlm/misc/000000028137.jpg b/vlmeval/vlm/misc/000000028137.jpg new file mode 100644 index 0000000..ea1f70d Binary files /dev/null and b/vlmeval/vlm/misc/000000028137.jpg differ diff --git a/vlmeval/vlm/misc/000000039769.jpg b/vlmeval/vlm/misc/000000039769.jpg new file mode 100644 index 0000000..e131e8e Binary files /dev/null and b/vlmeval/vlm/misc/000000039769.jpg differ diff --git a/vlmeval/vlm/misc/Idefics.jpg b/vlmeval/vlm/misc/Idefics.jpg new file mode 100644 index 0000000..3bf337e Binary files /dev/null and b/vlmeval/vlm/misc/Idefics.jpg differ diff --git a/vlmeval/vlm/misc/blip2_instruct_vicuna13b.yaml b/vlmeval/vlm/misc/blip2_instruct_vicuna13b.yaml new file mode 100644 index 0000000..a7cebe5 --- /dev/null +++ b/vlmeval/vlm/misc/blip2_instruct_vicuna13b.yaml @@ -0,0 +1,43 @@ + # Copyright (c) 2022, salesforce.com, inc. + # All rights reserved. + # SPDX-License-Identifier: BSD-3-Clause + # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + +model: + arch: instruct_vicuna13b + load_finetuned: False + load_pretrained: True + + pretrained: "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/InstructBLIP/instruct_blip_vicuna13b_trimmed.pth" + finetuned: "" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + freeze_vit: True + + # Q-Former + num_query_token: 32 + + # path to Vicuna checkpoint + llm_model: "Please set the path to your vicuna-13b-v1.1" + + # generation configs + prompt: "" + + +preprocess: + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" diff --git a/vlmeval/vlm/misc/blip2_instruct_vicuna7b.yaml b/vlmeval/vlm/misc/blip2_instruct_vicuna7b.yaml new file mode 100644 index 0000000..2a57a02 --- /dev/null +++ b/vlmeval/vlm/misc/blip2_instruct_vicuna7b.yaml @@ -0,0 +1,43 @@ + # Copyright (c) 2022, salesforce.com, inc. + # All rights reserved. + # SPDX-License-Identifier: BSD-3-Clause + # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + +model: + arch: instruct_vicuna7b + load_finetuned: False + load_pretrained: True + + pretrained: "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/InstructBLIP/instruct_blip_vicuna7b_trimmed.pth" + finetuned: "" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + freeze_vit: True + + # Q-Former + num_query_token: 32 + + # path to Vicuna checkpoint + llm_model: "Please set the path to your vicuna-7b-v1.1" + + # generation configs + prompt: "" + + +preprocess: + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" diff --git a/vlmeval/vlm/misc/minigpt4_13b_eval.yaml b/vlmeval/vlm/misc/minigpt4_13b_eval.yaml new file mode 100644 index 0000000..ae3d08b --- /dev/null +++ b/vlmeval/vlm/misc/minigpt4_13b_eval.yaml @@ -0,0 +1,37 @@ +model: + arch: minigpt4 + model_type: pretrain_vicuna_7b + max_txt_len: 160 + end_sym: "###" + low_resource: True + prompt_template: '###Human: {} ###Assistant: ' + ckpt: "please set this value to the path of pretrained checkpoint" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + freeze_vit: True + freeze_qformer: True + + # Q-Former + num_query_token: 32 + + # generation configs + prompt: "" + + llama_model: "please set this value to the path of vicuna-13b-v0" + +datasets: + cc_sbu_align: + vis_processor: + train: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + +run: + task: image_text_pretrain diff --git a/vlmeval/vlm/misc/minigpt4_7b_eval.yaml b/vlmeval/vlm/misc/minigpt4_7b_eval.yaml new file mode 100644 index 0000000..062c93a --- /dev/null +++ b/vlmeval/vlm/misc/minigpt4_7b_eval.yaml @@ -0,0 +1,38 @@ +model: + arch: minigpt4 + model_type: pretrain_vicuna_7b + max_txt_len: 160 + end_sym: "###" + low_resource: True + prompt_template: '###Human: {} ###Assistant: ' + ckpt: "please set this value to the path of pretrained checkpoint" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + freeze_vit: True + freeze_qformer: True + + # Q-Former + num_query_token: 32 + + # generation configs + prompt: "" + + llama_model: "please set this value to the path of vicuna-7b-v0" + + +datasets: + cc_sbu_align: + vis_processor: + train: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + +run: + task: image_text_pretrain diff --git a/vlmeval/vlm/misc/minigptv2_eval.yaml b/vlmeval/vlm/misc/minigptv2_eval.yaml new file mode 100644 index 0000000..32815e1 --- /dev/null +++ b/vlmeval/vlm/misc/minigptv2_eval.yaml @@ -0,0 +1,36 @@ +model: + arch: minigpt_v2 + model_type: pretrain + max_txt_len: 160 + end_sym: "" + low_resource: True + prompt_template: '[INST] {} [/INST]' + ckpt: "please set this value to the path of pretrained checkpoint" + lora_r: 64 + lora_alpha: 16 + + # vit encoder + image_size: 448 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + freeze_vit: True + + # generation configs + prompt: "" + + # LLM + llama_model: "please set this value to the path of llama2-chat-7b" + +datasets: + cc_sbu_align: + vis_processor: + train: + name: "blip2_image_eval" + image_size: 448 + text_processor: + train: + name: "blip_caption" + +run: + task: image_text_pretrain diff --git a/vlmeval/vlm/mmalaya.py b/vlmeval/vlm/mmalaya.py new file mode 100644 index 0000000..1bd95fa --- /dev/null +++ b/vlmeval/vlm/mmalaya.py @@ -0,0 +1,64 @@ +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer +import warnings +from PIL import Image +from .base import BaseModel + + +class MMAlaya(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='DataCanvas/MMAlaya', **kwargs): + assert model_path is not None + self.model_path = model_path + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained(model_path, device_map='cpu', trust_remote_code=True).eval() + # need initialize tokenizer + model.initialize_tokenizer(self.tokenizer) + self.model = model.cuda() + + self.kwargs = kwargs + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + torch.cuda.empty_cache() + + def generate_inner(self, message, dataset=None): + # read image + prompt, image_path = self.message_to_promptimg(message) + image = Image.open(image_path).convert('RGB') + # tokenize prompt, and proprecess image + input_ids, image_tensor, stopping_criteria = self.model.prepare_for_inference( + prompt, + self.tokenizer, + image, + return_tensors='pt') + with torch.inference_mode(): + output_ids = self.model.generate( + inputs=input_ids.cuda(), + images=image_tensor.cuda(), + do_sample=False, + max_new_tokens=512, + num_beams=1, + use_cache=True, + stopping_criteria=[stopping_criteria], + ) + # truncate input_ids in generate_ids and then decode to text + input_token_len = input_ids.shape[1] + response = self.tokenizer.batch_decode( + output_ids[:, input_token_len:].cpu(), + skip_special_tokens=True, + clean_up_tokenization_spaces=False + )[0].strip() + return response + + +if __name__ == '__main__': + model = MMAlaya() + response = model.generate(['./assets/apple.jpg', '请详细描述一下这张图片。']) + print(response) + +""" +export PYTHONPATH=$PYTHONPATH:/tmp/VLMEvalKit +CUDA_VISIBLE_DEVICES=0 python vlmeval/vlm/mmalaya.py +""" diff --git a/vlmeval/vlm/monkey.py b/vlmeval/vlm/monkey.py new file mode 100644 index 0000000..6fc3a51 --- /dev/null +++ b/vlmeval/vlm/monkey.py @@ -0,0 +1,165 @@ +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer +import warnings +from .base import BaseModel +from ..utils import DATASET_TYPE + + +class Monkey(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='echo840/Monkey', **kwargs): + assert model_path is not None + self.model_path = model_path + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained(model_path, device_map='cpu', trust_remote_code=True).eval() + self.model = model.cuda() + self.kwargs = kwargs + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + torch.cuda.empty_cache() + + def generate_vanilla(self, image_path, prompt): + cur_prompt = f'{image_path} {prompt} Answer: ' + input_ids = self.tokenizer(cur_prompt, return_tensors='pt', padding='longest') + attention_mask = input_ids.attention_mask + input_ids = input_ids.input_ids + + output_ids = self.model.generate( + input_ids=input_ids.cuda(), + attention_mask=attention_mask.cuda(), + do_sample=False, + num_beams=1, + max_new_tokens=512, + min_new_tokens=1, + length_penalty=1, + num_return_sequences=1, + output_hidden_states=True, + use_cache=True, + pad_token_id=self.tokenizer.eod_id, + eos_token_id=self.tokenizer.eod_id, + ) + response = self.tokenizer.decode( + output_ids[0][input_ids.size(1):].cpu(), + skip_special_tokens=True + ).strip() + return response + + def generate_multichoice(self, image_path, prompt): + cur_prompt = f'{image_path} \n {prompt} Answer: ' + input_ids = self.tokenizer(cur_prompt, return_tensors='pt', padding='longest') + attention_mask = input_ids.attention_mask + input_ids = input_ids.input_ids + + output_ids = self.model.generate( + input_ids=input_ids.cuda(), + attention_mask=attention_mask.cuda(), + do_sample=False, + num_beams=1, + max_new_tokens=10, + min_new_tokens=1, + length_penalty=1, + num_return_sequences=1, + output_hidden_states=True, + use_cache=True, + pad_token_id=self.tokenizer.eod_id, + eos_token_id=self.tokenizer.eod_id, + ) + response = self.tokenizer.decode( + output_ids[0][input_ids.size(1):].cpu(), + skip_special_tokens=True + ).strip() + return response + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + if dataset is None: + return self.generate_vanilla(image_path, prompt) + assert isinstance(dataset, str) + if DATASET_TYPE(dataset) == 'multi-choice' or DATASET_TYPE(dataset) == 'Y/N' or dataset == 'HallusionBench': + return self.generate_multichoice(image_path, prompt) + else: + return self.generate_vanilla(image_path, prompt) + + +class MonkeyChat(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='echo840/Monkey-Chat', **kwargs): + assert model_path is not None + self.model_path = model_path + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained(model_path, device_map='cpu', trust_remote_code=True).eval() + self.model = model.cuda() + self.kwargs = kwargs + + self.tokenizer.padding_side = 'left' + self.tokenizer.pad_token_id = self.tokenizer.eod_id + + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + torch.cuda.empty_cache() + + def generate_vanilla(self, image_path, prompt): + cur_prompt = f'{image_path} {prompt} Answer: ' + input_ids = self.tokenizer(cur_prompt, return_tensors='pt', padding='longest') + attention_mask = input_ids.attention_mask + input_ids = input_ids.input_ids + + output_ids = self.model.generate( + input_ids=input_ids.cuda(), + attention_mask=attention_mask.cuda(), + do_sample=False, + num_beams=1, + max_new_tokens=512, + min_new_tokens=1, + length_penalty=1, + num_return_sequences=1, + output_hidden_states=True, + use_cache=True, + pad_token_id=self.tokenizer.eod_id, + eos_token_id=self.tokenizer.eod_id, + ) + response = self.tokenizer.decode( + output_ids[0][input_ids.size(1):].cpu(), + skip_special_tokens=True + ).strip() + return response + + def generate_multichoice(self, image_path, prompt): + cur_prompt = f'{image_path} \n {prompt} Answer: ' + input_ids = self.tokenizer(cur_prompt, return_tensors='pt', padding='longest') + attention_mask = input_ids.attention_mask + input_ids = input_ids.input_ids + + output_ids = self.model.generate( + input_ids=input_ids.cuda(), + attention_mask=attention_mask.cuda(), + do_sample=False, + num_beams=1, + max_new_tokens=10, + min_new_tokens=1, + length_penalty=1, + num_return_sequences=1, + output_hidden_states=True, + use_cache=True, + pad_token_id=self.tokenizer.eod_id, + eos_token_id=self.tokenizer.eod_id, + ) + response = self.tokenizer.decode( + output_ids[0][input_ids.size(1):].cpu(), + skip_special_tokens=True + ).strip() + return response + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + if dataset is None: + return self.generate_vanilla(image_path, prompt) + assert isinstance(dataset, str) + if DATASET_TYPE(dataset) == 'multi-choice' or DATASET_TYPE(dataset) == 'Y/N' or dataset == 'HallusionBench': + return self.generate_multichoice(image_path, prompt) + else: + return self.generate_vanilla(image_path, prompt) diff --git a/vlmeval/vlm/mplug_owl2.py b/vlmeval/vlm/mplug_owl2.py new file mode 100644 index 0000000..45a8222 --- /dev/null +++ b/vlmeval/vlm/mplug_owl2.py @@ -0,0 +1,126 @@ +import sys +import torch +from PIL import Image +from .base import BaseModel +from ..smp import * +from ..utils import DATASET_TYPE + + +class mPLUG_Owl2(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def __init__(self, model_path='MAGAer13/mplug-owl2-llama2-7b', **kwargs): + try: + from mplug_owl2.model.builder import load_pretrained_model + from mplug_owl2.mm_utils import get_model_name_from_path + except: + warnings.warn('Please install mPLUG_Owl2 before using mPLUG_Owl2. ') + sys.exit(-1) + + model_name = get_model_name_from_path(model_path) + tokenizer, model, image_processor, context_len = load_pretrained_model( + model_path, None, model_name, load_8bit=False, load_4bit=False, device='cpu') + + self.model = model.cuda() + self.device = self.model.device + self.image_processor = image_processor + tokenizer.padding_side = 'left' + tokenizer.pad_token_id = tokenizer.eos_token_id + self.tokenizer = tokenizer + self.context_len = context_len + + kwargs_default = dict( + max_new_tokens=512, do_sample=False, num_beams=1, + min_new_tokens=1, length_penalty=1, num_return_sequences=1) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def use_custom_prompt(self, dataset): + assert dataset is not None + if listinstr(['MMMU'], dataset): + return False + if DATASET_TYPE(dataset) == 'multi-choice' or dataset == 'MMVet': + return True + return False + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + question = line['question'] + if dataset == 'MMVet': + prompt = question + '\nAnswer the question directly. ' + elif DATASET_TYPE(dataset) == 'multi-choice': + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + options_prompt = '' + for key, item in options.items(): + options_prompt += f'{key}. {item}\n' + + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + prompt = f'Hint: {hint}\n' if hint is not None else '' + prompt += f'{question}\n' + prompt += ( + f'{options_prompt}\nAnswer with the option’s letter from the given choices directly. ' + if len(options) else 'Answer the question directly. ' + ) + else: + raise NotImplementedError + + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + return message + + def generate_inner(self, message, dataset=None): + from mplug_owl2.constants import IMAGE_TOKEN_INDEX + from mplug_owl2.mm_utils import process_images, tokenizer_image_token + kwargs = cp.deepcopy(self.kwargs) + if dataset in ['MMVet', 'LLaVABench']: + kwargs['length_penalty'] = 0 + elif dataset is not None and DATASET_TYPE(dataset) == 'VQA': + kwargs['length_penalty'] = 0 + elif dataset is not None and DATASET_TYPE(dataset) == 'multi-choice': + kwargs['max_new_tokens'] = 10 + num_images = len([x for x in message if x['type'] == 'image']) + assert num_images >= 0 + prompt_full = 'USER: ' + images = [] + if num_images == 1: + prompt, image = self.message_to_promptimg(message) + prompt_full += f'<|image|>{prompt} \nASSISTANT: ' + images.append(image) + else: + for msg in message: + if msg['type'] == 'image': + images.append(msg['value']) + prompt_full += '<|image|>' + elif msg['type'] == 'text': + prompt_full += msg['value'] + prompt_full += '\nASSISTANT: ' + + def preproc_image(fname): + image = Image.open(fname).convert('RGB') + max_edge = max(image.size) + image = image.resize((max_edge, max_edge)) + return image + images = [preproc_image(fname) for fname in images] + image_tensor = process_images(images, self.image_processor) + image_tensor = image_tensor.to(self.device, dtype=torch.float16) + input_ids = tokenizer_image_token( + prompt_full, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(self.device) + + with torch.inference_mode(): + output_ids = self.model.generate( + input_ids=input_ids, + images=image_tensor, + output_hidden_states=True, + use_cache=True, + **kwargs) + answer = self.tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip() + return answer.split('')[0] diff --git a/vlmeval/vlm/omnilmm.py b/vlmeval/vlm/omnilmm.py new file mode 100644 index 0000000..26f4a2e --- /dev/null +++ b/vlmeval/vlm/omnilmm.py @@ -0,0 +1,152 @@ +import sys +import torch +from PIL import Image +from transformers import AutoTokenizer + +from .base import BaseModel +from ..smp import * +from ..utils import DATASET_TYPE + + +DEFAULT_IMAGE_TOKEN = '' +DEFAULT_IMAGE_PATCH_TOKEN = '' +DEFAULT_IM_START_TOKEN = '' +DEFAULT_IM_END_TOKEN = '' + + +def init_omni_lmm(model_path): + from omnilmm.model.omnilmm import OmniLMMForCausalLM + from omnilmm.utils import disable_torch_init + from omnilmm.model.utils import build_transform + + torch.backends.cuda.matmul.allow_tf32 = True + disable_torch_init() + tokenizer = AutoTokenizer.from_pretrained(model_path, model_max_length=2048) + + model = OmniLMMForCausalLM.from_pretrained(model_path, tune_clip=True, torch_dtype=torch.bfloat16, device_map='cpu') + model = model.to(device='cuda', dtype=torch.bfloat16) + + image_processor = build_transform(is_train=False, input_size=model.model.config.image_size, std_mode='OPENAI_CLIP') + + mm_use_im_start_end = getattr(model.config, 'mm_use_im_start_end', False) + assert mm_use_im_start_end + + tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True) + + vision_config = model.model.vision_config + vision_config.im_patch_token = tokenizer.convert_tokens_to_ids( + [DEFAULT_IMAGE_PATCH_TOKEN])[0] + vision_config.use_im_start_end = mm_use_im_start_end + vision_config.im_start_token, vision_config.im_end_token = tokenizer.convert_tokens_to_ids( + [DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]) + image_token_len = model.model.config.num_query + + return model, image_processor, image_token_len, tokenizer + + +def expand_question_into_multimodal(question_text, image_token_len, im_st_token, im_ed_token, im_patch_token): + if '' in question_text[0]['content']: + question_text[0]['content'] = question_text[0]['content'].replace( + '', im_st_token + im_patch_token * image_token_len + im_ed_token) + else: + question_text[0]['content'] = im_st_token + im_patch_token * \ + image_token_len + im_ed_token + '\n' + question_text[0]['content'] + return question_text + + +def wrap_question_for_omni_lmm(question, image_token_len, tokenizer): + from omnilmm.train.train_utils import omni_preprocess + question = expand_question_into_multimodal( + question, image_token_len, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, DEFAULT_IMAGE_PATCH_TOKEN) + + conversation = question + data_dict = omni_preprocess(sources=[conversation], tokenizer=tokenizer, generation=True) + + data_dict = dict(input_ids=data_dict['input_ids'][0], labels=data_dict['labels'][0]) + return data_dict + + +class OmniLMM12B(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def __init__(self, model_path, root, **kwargs) -> None: + sys.path.append(root) + model, img_processor, image_token_len, tokenizer = init_omni_lmm(model_path) + self.model = model + self.image_token_len = image_token_len + self.image_transform = img_processor + self.tokenizer = tokenizer + self.model.eval() + default_kwargs = dict( + max_new_tokens=512, + do_sample=False, + output_scores=True, + return_dict_in_generate=True, + repetition_penalty=1.1) + default_kwargs.update(kwargs) + self.kwargs = default_kwargs + torch.cuda.empty_cache() + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + try: + image = Image.open(image_path).convert('RGB') + except: + logger = get_logger('OmniLMM Inference') + logger.error('Image Decode Error') + return 'Image Decode Error' + + msgs = [dict(role='user', content=prompt)] + input_ids = wrap_question_for_omni_lmm( + msgs, self.image_token_len, self.tokenizer)['input_ids'] + input_ids = torch.as_tensor(input_ids) + image = self.image_transform(image) + + with torch.inference_mode(): + output = self.model.generate_vllm( + input_ids=input_ids.unsqueeze(0).cuda(), + images=image.unsqueeze(0).half().cuda(), + **self.kwargs) + + response = self.tokenizer.decode( + output.sequences[0], skip_special_tokens=True) + response = response.strip() + return response + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + question = line['question'] + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + options_prompt = 'Options:\n' + for key, item in options.items(): + options_prompt += f'{key}. {item}\n' + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + prompt = '' + if hint is not None: + prompt += f'Hint: {hint}\n' + prompt += f'{question}\n' + if len(options): + prompt += options_prompt + prompt = """ +Study the image carefully and pick the option associated with the correct answer. +Focus solely on selecting the option and avoid including any other content.\n +""" + prompt + + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + return message diff --git a/vlmeval/vlm/open_flamingo.py b/vlmeval/vlm/open_flamingo.py new file mode 100644 index 0000000..2ac354a --- /dev/null +++ b/vlmeval/vlm/open_flamingo.py @@ -0,0 +1,99 @@ +import sys +import torch +from PIL import Image +import os.path as osp +import warnings +from .base import BaseModel +from ..smp import splitlen, get_cache_path +from huggingface_hub import snapshot_download + + +class OpenFlamingo(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = True + + def __init__(self, + name, + mpt_pth=None, + ckpt_pth=None, + **kwargs): + + if mpt_pth is None: + warnings.warn( + 'Please set `mpt_pth` to the directory of MPT-7B, which is cloned from here: ' + 'https://huggingface.co/mosaicml/mpt-7b. ' + ) + sys.exit(-1) + if ckpt_pth is None: + warnings.warn( + 'Please set `ckpt_pth` to the openflamingo ckpt, which is the `checkpoint.pt` file downloaded ' + 'from: https://huggingface.co/openflamingo/OpenFlamingo-9B-vitl-mpt7b/tree/main. ' + ) + sys.exit(-1) + else: + if osp.exists(ckpt_pth): + if ckpt_pth.endswith('checkpoint.pt'): + pass + elif osp.isdir(ckpt_pth): + ckpt_pth = osp.join(ckpt_pth, 'checkpoint.pt') + if not osp.exists(ckpt_pth): + sys.exit(-1) + elif splitlen(ckpt_pth, '/') == 2: + cache_path = get_cache_path(ckpt_pth) + if cache_path is None: + snapshot_download(ckpt_pth) + cache_path = get_cache_path(ckpt_pth) + if cache_path is None: + sys.exit(-1) + else: + ckpt_pth = osp.join(cache_path, 'checkpoint.pt') + + self.name = name + assert name in ['v2'] + self.mpt_pth = mpt_pth + try: + from open_flamingo import create_model_and_transforms + except: + raise ImportError('Please first install open_flamingo to use OpenFlamingo') + model, image_processor, tokenizer = create_model_and_transforms( + clip_vision_encoder_path='ViT-L-14', + clip_vision_encoder_pretrained='openai', + lang_encoder_path=mpt_pth, + tokenizer_path=mpt_pth, + cross_attn_every_n_layers=4) + ckpt = torch.load(ckpt_pth) + model.load_state_dict(ckpt, strict=False) + torch.cuda.empty_cache() + self.model = model.eval().cuda() + self.tokenizer = tokenizer + self.tokenizer.padding_side = 'left' + self.image_proc = image_processor + + kwargs_default = dict(max_new_tokens=512, num_beams=3) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def generate_inner(self, message, dataset=None): + vision_x = [] + prompt = '' + for msg in message: + if msg['type'] == 'image': + img = Image.open(msg['value']) + vision_x.append(self.image_proc(img).unsqueeze(0)) + prompt += '' + elif msg['type'] == 'text': + prompt += msg['value'] + prompt += 'Answer: ' + vision_x = torch.cat(vision_x, dim=0) if len(vision_x) > 1 else vision_x[0] + vision_x = vision_x.unsqueeze(1).unsqueeze(0) + lang_x = self.tokenizer([prompt], return_tensors='pt') + generated_text = self.model.generate( + vision_x=vision_x.cuda(), + lang_x=lang_x['input_ids'].cuda(), + attention_mask=lang_x['attention_mask'].cuda(), + **self.kwargs) + generated_text = self.tokenizer.decode(generated_text[0]) + text = generated_text[len(prompt):].split('<|endofchunk|>')[0] + return text diff --git a/vlmeval/vlm/pandagpt.py b/vlmeval/vlm/pandagpt.py new file mode 100644 index 0000000..877ed9e --- /dev/null +++ b/vlmeval/vlm/pandagpt.py @@ -0,0 +1,61 @@ +import sys +import torch +import os.path as osp +import warnings +from .base import BaseModel + + +class PandaGPT(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def __init__(self, name, root=None, **kwargs): + if root is None: + warnings.warn('Please set `root` to PandaGPT code directory, which is cloned from here: ') + sys.exit(-1) + + assert name == 'PandaGPT_13B' + self.name = name + sys.path.append(osp.join(root, 'code')) + try: + from model.openllama import OpenLLAMAPEFTModel + except: + raise ImportError( + 'Please first install PandaGPT and set the root path to use PandaGPT, ' + 'which is cloned from here: https://github.com/yxuansu/PandaGPT. ' + ) + self.args = { + 'model': 'openllama_peft', + 'imagebind_ckpt_path': osp.join(root, 'pretrained_ckpt/imagebind_ckpt'), + 'vicuna_ckpt_path': osp.join(root, 'pretrained_ckpt/vicuna_ckpt/13b_v0'), + 'delta_ckpt_path': osp.join(root, 'pretrained_ckpt/pandagpt_ckpt/13b/pytorch_model.pt'), + 'stage': 2, + 'max_tgt_len': 512, + 'lora_r': 32, + 'lora_alpha': 32, + 'lora_dropout': 0.1, + } + model = OpenLLAMAPEFTModel(**self.args) + delta_ckpt = torch.load(self.args['delta_ckpt_path'], map_location=torch.device('cpu')) + model.load_state_dict(delta_ckpt, strict=False) + torch.cuda.empty_cache() + self.model = model.eval().half().cuda() + kwargs_default = {'top_p': 0.9, 'do_sample': False, 'max_tgt_len': 128, 'temperature': 0.001} + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + struct = { + 'prompt': prompt, + 'image_paths': [image_path], + 'audio_paths': [], + 'video_paths': [], + 'thermal_paths': [], + 'modality_embeds': [] + } + struct.update(self.kwargs) + resp = self.model.generate(struct) + return resp diff --git a/vlmeval/vlm/qwen_vl.py b/vlmeval/vlm/qwen_vl.py new file mode 100644 index 0000000..00334da --- /dev/null +++ b/vlmeval/vlm/qwen_vl.py @@ -0,0 +1,94 @@ +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer +import warnings +import copy as cp +from .base import BaseModel +from ..smp import isimg, listinstr +from ..utils import DATASET_TYPE + + +class QwenVL(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = True + + def __init__(self, model_path='Qwen/Qwen-VL', **kwargs): + assert model_path is not None + self.model_path = model_path + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + tokenizer.padding_side = 'left' + tokenizer.pad_token_id = tokenizer.eod_id + self.tokenizer = tokenizer + self.model = AutoModelForCausalLM.from_pretrained(model_path, device_map='cuda', trust_remote_code=True).eval() + default_kwargs = dict( + do_sample=False, + num_beams=1, + max_new_tokens=512, + min_new_tokens=1, + num_return_sequences=1, + use_cache=True, + output_hidden_states=True, + pad_token_id=tokenizer.eod_id, + eos_token_id=tokenizer.eod_id) + default_kwargs.update(kwargs) + self.kwargs = default_kwargs + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + torch.cuda.empty_cache() + + def adjust_kwargs(self, dataset): + kwargs = cp.deepcopy(self.kwargs) + if DATASET_TYPE(dataset) in ['multi-choice', 'Y/N']: + kwargs['max_new_tokens'] = 32 + elif DATASET_TYPE(dataset) == 'Caption' and 'COCO' in dataset: + kwargs['max_new_tokens'] = 32 + elif DATASET_TYPE(dataset) == 'VQA': + if listinstr(['OCRVQA', 'ChartQA', 'DocVQA'], dataset): + kwargs['max_new_tokens'] = 100 + elif listinstr(['TextVQA'], dataset): + kwargs['max_new_tokens'] = 10 + return kwargs + + def generate_inner(self, message, dataset=None): + if dataset is not None: + kwargs = self.adjust_kwargs(dataset) + else: + kwargs = self.kwargs + prompt = '' + for s in message: + if s['type'] == 'image': + prompt += f'{s["value"]}' + elif s['type'] == 'text': + prompt += s['value'] + if dataset is not None and DATASET_TYPE(dataset) == 'VQA': + prompt += ' Answer:' + encoded = self.tokenizer([prompt], return_tensors='pt', padding='longest') + input_ids = encoded.input_ids.to('cuda') + attention_mask = encoded.attention_mask.to('cuda') + + pred = self.model.generate( + input_ids=input_ids, + attention_mask=attention_mask, + **kwargs) + answer = self.tokenizer.decode(pred[0][input_ids.size(1):].cpu(), skip_special_tokens=True).strip() + return answer + + +class QwenVLChat(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = True + + def __init__(self, model_path='Qwen/Qwen-VL-Chat', **kwargs): + assert model_path is not None + self.model_path = model_path + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + self.model = AutoModelForCausalLM.from_pretrained(model_path, device_map='cuda', trust_remote_code=True).eval() + torch.cuda.empty_cache() + self.kwargs = kwargs + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def generate_inner(self, message, dataset=None): + vl_list = [{'image': s['value']} if s['type'] == 'image' else {'text': s['value']} for s in message] + query = self.tokenizer.from_list_format(vl_list) + response, _ = self.model.chat(self.tokenizer, query=query, history=None, **self.kwargs) + return response diff --git a/vlmeval/vlm/transcore_m.py b/vlmeval/vlm/transcore_m.py new file mode 100644 index 0000000..7e4b6a7 --- /dev/null +++ b/vlmeval/vlm/transcore_m.py @@ -0,0 +1,162 @@ +import sys +import torch +from abc import abstractproperty +from .base import BaseModel +from ..smp import * +from ..utils import DATASET_TYPE +from transformers import AutoTokenizer, BitsAndBytesConfig + + +class TransCoreM(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def load_pretrained_model(self, model_path, load_8bit=False, load_4bit=False, revision='main'): + from transcorem.model import TransCoreMQWenForCausalLM + from transcorem.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN + import transcorem.config_param as config_param + kwargs = {'revision': revision} + if load_8bit: + kwargs['load_in_8bit'] = True + elif load_4bit: + kwargs['load_in_4bit'] = True + kwargs['quantization_config'] = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4' + ) + else: + kwargs['torch_dtype'] = torch.float16 + + config_param.model_path = model_path + tokenizer = AutoTokenizer.from_pretrained( + model_path, use_fast=False, revision=revision, trust_remote_code=True) + model = TransCoreMQWenForCausalLM.from_pretrained( + model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs) + + image_processor = None + mm_use_im_start_end = getattr(model.config, 'mm_use_im_start_end', False) + mm_use_im_patch_token = getattr(model.config, 'mm_use_im_patch_token', True) + if mm_use_im_patch_token: + tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True) + if mm_use_im_start_end: + tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True) + model.resize_token_embeddings(len(tokenizer)) + + vision_tower = model.get_vision_tower() + if not vision_tower.is_loaded: + vision_tower.load_model() + vision_tower.to(device='cpu', dtype=torch.float16) + image_processor = vision_tower.image_processor + + if hasattr(model.config, 'max_sequence_length'): + context_len = model.config.max_sequence_length + else: + context_len = 2048 + + return tokenizer, model, image_processor, context_len + + def __init__(self, + root=None, + revision='main', + **kwargs): + + self.root = root + self.revision = revision + sys.path.append(root) + + model_path = 'PCIResearch/TransCore-M' + assert osp.exists(model_path) or splitlen(model_path) == 2 + self.tokenizer, self.model, self.image_processor, self.context_len = self.load_pretrained_model( + model_path=model_path, revision=revision) + self.model = self.model.cuda() + print('==============conv_mode: transcorem_v1') + self.conv_mode = 'transcorem_v1' + + kwargs_default = dict(do_sample=False, temperature=0.0, max_new_tokens=512, top_p=None, num_beams=1) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + question = line['question'] + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + prompt = question + + if len(options): + prompt += ( + '\n请直接回答选项字母。' if cn_string(prompt) else + "\nAnswer with the option's letter from the given choices directly." + ) + else: + prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.' + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=f) for f in tgt_path]) + return message + + def generate_inner(self, message, dataset=None): + from transcorem.mm_utils import highres_process_images, tokenizer_image_token, KeywordsStoppingCriteria + from transcorem.constants import ( + IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN) + from transcorem.conversation import conv_templates, SeparatorStyle + + prompt, image_path = self.message_to_promptimg(message) + image = Image.open(image_path).convert('RGB') + args = abstractproperty() + args.image_aspect_ratio = 'pad' + image_patches = highres_process_images(image, self.image_processor, args, base_reso=336) + image_patches = [patch.unsqueeze(0).to('cuda', dtype=torch.float16) for patch in image_patches] + if self.model.config.mm_use_im_start_end: + inp = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + prompt + else: + inp = DEFAULT_IMAGE_TOKEN + '\n' + prompt + + conv = conv_templates[self.conv_mode].copy() + conv.append_message(conv.roles[0], inp) + conv.append_message(conv.roles[1], None) + prompt_conv = conv.get_prompt() + input_ids = tokenizer_image_token(prompt_conv, self.tokenizer, IMAGE_TOKEN_INDEX, + return_tensors='pt').unsqueeze(0).cuda() + stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 + keywords = [stop_str] + stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids) + with torch.inference_mode(): + output_ids = self.model.generate( + input_ids, + images=image_patches, + use_cache=True, + stopping_criteria=[stopping_criteria], + **self.kwargs) + + input_token_len = input_ids.shape[1] + n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item() + if n_diff_input_output > 0: + print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids') + outputs = self.tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0] + outputs = outputs.strip() + if outputs.endswith(stop_str): + outputs = outputs[:-len(stop_str)] + outputs = outputs.strip() + return outputs diff --git a/vlmeval/vlm/visualglm.py b/vlmeval/vlm/visualglm.py new file mode 100644 index 0000000..76f077a --- /dev/null +++ b/vlmeval/vlm/visualglm.py @@ -0,0 +1,36 @@ +import warnings +from .base import BaseModel +from ..smp import * + + +class VisualGLM(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='THUDM/visualglm-6b', **kwargs): + try: + import sat + except: + warnings.warn('Please install SwissArmyTransformer to use VisualGLM') + assert model_path is not None + self.model_path = model_path + + from transformers import AutoModel + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = AutoModel.from_pretrained(model_path, trust_remote_code=True).half().cuda() + self.model = model + self.kwargs = kwargs + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + output, _ = self.model.chat( + image_path=image_path, + tokenizer=self.tokenizer, + query=prompt, + history=[], + **self.kwargs + ) + return output diff --git a/vlmeval/vlm/xcomposer/__init__.py b/vlmeval/vlm/xcomposer/__init__.py new file mode 100644 index 0000000..1d8237f --- /dev/null +++ b/vlmeval/vlm/xcomposer/__init__.py @@ -0,0 +1,6 @@ +from .sharecaptioner import ShareCaptioner +from .xcomposer import XComposer +from .xcomposer2 import XComposer2 +from .xcomposer2_4KHD import XComposer2_4KHD + +__all__ = ['ShareCaptioner', 'XComposer', 'XComposer2', 'XComposer2_4KHD'] diff --git a/vlmeval/vlm/xcomposer/sharecaptioner.py b/vlmeval/vlm/xcomposer/sharecaptioner.py new file mode 100644 index 0000000..8823816 --- /dev/null +++ b/vlmeval/vlm/xcomposer/sharecaptioner.py @@ -0,0 +1,92 @@ +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer +from ..base import BaseModel +from ...smp import * +from ...utils import DATASET_TYPE + + +class ShareCaptioner(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='Lin-Chen/ShareCaptioner', **kwargs): + assert model_path is not None + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + self.model = AutoModelForCausalLM.from_pretrained( + model_path, device_map='cuda', trust_remote_code=True).eval() + self.model.tokenizer = tokenizer + self.model.cuda() + self.model.half() + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + if dataset is not None and DATASET_TYPE(dataset) == 'multi-choice': + question = line['question'] + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + option_candidate = string.ascii_uppercase + options = { + cand: line[cand] + for cand in option_candidate + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + prompt = question + + if not cn_string(prompt): + prompt = prompt + '\n' + "Answer with the option's letter from the given choices directly." + else: + prompt = prompt + '\n' + '请直接回答选项字母。' + else: + prompt = line['question'] + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + return message + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + seg1 = '<|User|>:' + seg2 = f'{prompt}{self.model.eoh}\n<|Bot|>:' + self.seg_emb1 = self.model.encode_text(seg1, add_special_tokens=True) + self.seg_emb2 = self.model.encode_text(seg2, add_special_tokens=False) + + image = Image.open(image_path).convert('RGB') + image = self.model.vis_processor(image).unsqueeze(0) + image = image.to(self.model.device) + tmp_bs = image.shape[0] + tmp_seg_emb1 = self.seg_emb1.repeat(tmp_bs, 1, 1) + tmp_seg_emb2 = self.seg_emb2.repeat(tmp_bs, 1, 1) + with torch.cuda.amp.autocast(): + with torch.no_grad(): + image = self.model.encode_img(image) + input_emb = torch.cat( + [tmp_seg_emb1, image, tmp_seg_emb2], dim=1) + out_embeds = self.model.internlm_model.generate( + inputs_embeds=input_emb, + max_length=500, + num_beams=3, + min_length=1, + do_sample=True, + repetition_penalty=1.5, + length_penalty=1.0, + temperature=1., + eos_token_id=self.model.tokenizer.eos_token_id, + num_return_sequences=1) + + for j, out in enumerate(out_embeds): + out[out == -1] = 2 + response = self.model.decode_text([out]) + return response diff --git a/vlmeval/vlm/xcomposer/xcomposer.py b/vlmeval/vlm/xcomposer/xcomposer.py new file mode 100644 index 0000000..3a93cff --- /dev/null +++ b/vlmeval/vlm/xcomposer/xcomposer.py @@ -0,0 +1,150 @@ +import torch +from transformers import AutoModel, AutoTokenizer +from transformers import StoppingCriteria, StoppingCriteriaList +from PIL import Image +from ..base import BaseModel +from ...smp import * + + +class StoppingCriteriaSub(StoppingCriteria): + def __init__(self, stops=[], encounters=1): + super().__init__() + self.stops = stops + + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor): + for stop in self.stops: + if torch.all((stop == input_ids[0][-len(stop):])).item(): + return True + + return False + + +from ...utils import DATASET_TYPE + + +class XComposer(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='internlm/internlm-xcomposer-vl-7b', **kwargs): + assert model_path is not None + self.model_path = model_path + + model = AutoModel.from_pretrained(self.model_path, device_map='cpu', trust_remote_code=True).cuda().eval() + tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True) + model.tokenizer = tokenizer + self.model = model + self.device = self.model.internlm_model.model.embed_tokens.weight.device + self.eoh = '' + self.eoa = '' + stop_words_ids = [ + torch.tensor([103027]).to(self.device), # end of human + torch.tensor([103028]).to(self.device), # end of bot + ] + default_kwargs = { + 'max_new_tokens': 512, 'num_beams': 5, 'do_sample': False, + 'min_length': 1, 'repetition_penalty': 1.5, 'length_penalty': 1.0 + } + default_kwargs.update(kwargs) + self.kwargs = default_kwargs + self.stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)]) + + def generate_inner(self, message, dataset=None): + if len(message) == 2: + if message[0]['type'] == 'text' and message[1]['type'] == 'image': + message = [message[1], message[0]] + kwargs = cp.deepcopy(self.kwargs) + if dataset is not None: + if DATASET_TYPE(dataset) == 'multi-choice': + kwargs['max_new_tokens'] = 5 + kwargs['num_beams'] = 5 + + with torch.cuda.amp.autocast(): + with torch.no_grad(): + prompt_embs = self.message_to_prompt_embs(message, dataset) + outputs = self.model.internlm_model.generate( + inputs_embeds=prompt_embs, + stopping_criteria=self.stopping_criteria, + **kwargs + ) + + output_token = outputs[0] + if output_token[0] == 0: + output_token = output_token[1:] + if output_token[0] == 1: + output_token = output_token[1:] + output_text = self.model.tokenizer.decode(output_token, add_special_tokens=False) + + output_text = output_text.split(self.model.eoa)[0] + output_text = output_text.split('<|Bot|>')[-1].strip() + return output_text + + def message_to_prompt_embs(self, message, dataset=None): + assert isinstance(message, list) + img_embeds = [] + prompt_full = '<|User|>: ' + for msg in message: + if msg['type'] == 'text': + prompt_full += msg['value'] + elif msg['type'] == 'image': + image = Image.open(msg['value']).convert('RGB') + image = self.model.vis_processor(image).unsqueeze(0).to(self.device) + img_embeds.append(self.model.encode_img(image)) + prompt_full += '' + + prompt_full += self.model.eoh + ' <|Bot|>: ' + if dataset is not None and DATASET_TYPE(dataset) == 'multi-choice': + prompt_full += 'Answer: The answer is ' + elif dataset is not None and DATASET_TYPE(dataset) in ['VQA', 'QA', 'Y/N']: + prompt_full += 'Answer: ' + + prompt_segs = prompt_full.split('') + assert len(prompt_segs) == len(img_embeds) + 1 + + prompt_seg_tokens = [ + self.model.tokenizer(seg, return_tensors='pt', add_special_tokens=(i == 0)).to(self.device).input_ids + for i, seg in enumerate(prompt_segs) + ] + prompt_seg_embs = [self.model.internlm_model.model.embed_tokens(seg) for seg in prompt_seg_tokens] + all_embeddings = [] + for i in range(len(img_embeds)): + all_embeddings.extend([prompt_seg_embs[i], img_embeds[i]]) + all_embeddings.append(prompt_seg_embs[-1]) + prompt_embs = torch.cat(all_embeddings, dim=1) + return prompt_embs + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice': + return True + return False + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + question = line['question'] + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + options_prompt = '' + for key, item in options.items(): + options_prompt += f'{key}. {item}\n' + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + context = 'N/A' if hint is None else hint + mid_prompt = 'Context: ' + context + '\nQuestion: ' + question + if len(options_prompt): + mid_prompt += '\nOptions: ' + options_prompt + + if len(options): + txt_prompt = 'Please answer this question by choosing the correct choice.' + else: + txt_prompt = 'Please answer this question directly. ' + prompt = txt_prompt + mid_prompt + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + return message diff --git a/vlmeval/vlm/xcomposer/xcomposer2.py b/vlmeval/vlm/xcomposer/xcomposer2.py new file mode 100644 index 0000000..c2807fe --- /dev/null +++ b/vlmeval/vlm/xcomposer/xcomposer2.py @@ -0,0 +1,213 @@ +import torch +import torchvision +from transformers import AutoModel, AutoTokenizer +from PIL import Image +from ..base import BaseModel +from ...smp import * +from ...utils import DATASET_TYPE +import re +pattern = re.compile(r'[A-Z]') + + +def __padding__(image): + width, height = image.size + tar = max(width, height) + top_padding = int((tar - height) / 2) + bottom_padding = tar - height - top_padding + left_padding = int((tar - width) / 2) + right_padding = tar - width - left_padding + image = torchvision.transforms.functional.pad(image, [left_padding, top_padding, right_padding, bottom_padding]) + return image + + +meta_instruction = """ +You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔). +- InternLM-XComposer (浦语·灵笔) is a multi-modality conversational language model that is developed by +Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless. +- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language +chosen by the user such as English and 中文. +- InternLM-XComposer (浦语·灵笔) is capable of comprehending and articulating responses effectively +based on the provided image. +""" + + +def model_gen(model, text, images, need_bos=True, padding=False, beams=3, max_token=500): + pt1 = 0 + embeds = [] + im_mask = [] + images = [images] + images_loc = [0] + for i, pts in enumerate(images_loc + [len(text)]): + subtext = text[pt1:pts] + if need_bos or len(subtext) > 0: + text_embeds = model.encode_text(subtext, add_special_tokens=need_bos) + embeds.append(text_embeds) + im_mask.append(torch.zeros(text_embeds.shape[:2]).cuda()) + need_bos = False + if i < len(images): + try: + image = Image.open(images[i]).convert('RGB') + except: + image = images[i].convert('RGB') + if padding: + image = __padding__(image) + image = model.vis_processor(image).unsqueeze(0).cuda() + image_embeds = model.encode_img(image) + embeds.append(image_embeds) + im_mask.append(torch.ones(image_embeds.shape[:2]).cuda()) + pt1 = pts + embeds = torch.cat(embeds, dim=1) + im_mask = torch.cat(im_mask, dim=1) + im_mask = im_mask.bool() + + outputs = model.generate( + inputs_embeds=embeds, + im_mask=im_mask, + temperature=1.0, + max_new_tokens=max_token, + num_beams=beams, + do_sample=False, + repetition_penalty=1.0) + + output_token = outputs[0] + if output_token[0] == 0 or output_token[0] == 1: + output_token = output_token[1:] + output_text = model.tokenizer.decode(output_token, add_special_tokens=False) + output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip() + return output_text + + +class XComposer2(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='internlm/internlm-xcomposer2-vl-7b', **kwargs): + assert model_path is not None + self.model_path = model_path + + model = AutoModel.from_pretrained(self.model_path, device_map='cpu', trust_remote_code=True).cuda().eval() + model.half() + tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True) + model.tokenizer = tokenizer + self.model = model + self.device = self.model.model.tok_embeddings.weight.device + + def generate_mme(self, image_path, text): + text = text.split('Please answer')[0].strip() + text = f'{text} Answer this question briefly' + text = f'[UNUSED_TOKEN_146]user\n{text}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + + return model_gen(self.model, text, image_path, need_bos=True, padding=True, beams=5) + + def generate_multichoice(self, image_path, text, dataset): + out = model_gen(self.model, text, image_path, need_bos=True, padding=False, beams=5, max_token=5) + if 'mmmu' in dataset.lower(): + return out + res = pattern.findall(out) + if len(res) == 0: + print('Error:', out) + res = 'Z' + return res[0] + + def generate_vqa(self, image_path, text): + out = model_gen(self.model, text, image_path, need_bos=True) + return out + + def generate_vanilla(self, image_path, text): + text = ( + '[UNUSED_TOKEN_146]system\n{}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]user\n{}' + 'Answer this question in detail.[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + ).format(meta_instruction, text) + out = model_gen(self.model, text, image_path, need_bos=True, max_token=500) + return out + + def generate_brief(self, image_path, text): + text = ( + '[UNUSED_TOKEN_146]user\nAnswer the question using a single word or phrase.{}' + '[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + ).format(text) + out = model_gen(self.model, text, image_path, need_bos=True, max_token=10) + return out + + def generate_driectly(self, image_path, text): + text = '[UNUSED_TOKEN_146]user\n{}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n'.format(text) + out = model_gen(self.model, text, image_path, need_bos=True, max_token=500) + return out + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + with torch.cuda.amp.autocast(): + if dataset is None: + return self.generate_vanilla(image_path, prompt) + assert isinstance(dataset, str) + if dataset == 'MME': + return self.generate_mme(image_path, prompt) + + elif listinstr(['hallu'], dataset.lower()): + return self.generate_brief(image_path, prompt) + + elif listinstr(['llava'], dataset.lower()): + return self.generate_vanilla(image_path, prompt) + + elif dataset is not None and DATASET_TYPE(dataset) == 'multi-choice': + return self.generate_multichoice(image_path, prompt, dataset) + + elif dataset is not None and DATASET_TYPE(dataset) == 'VQA': + return self.generate_vqa(image_path, prompt) + + else: + return self.generate_vanilla(image_path, prompt) + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice' or DATASET_TYPE(dataset) == 'VQA': + return True + return False + + def build_mcqa(self, line): + question = line['question'] + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + img_prompt = '[UNUSED_TOKEN_146]user\n' + if len(options): + options_prompt = '' + for key, item in options.items(): + options_prompt += f'{key}. {item} ' + options_prompt = options_prompt.strip() + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + + context = 'N/A' if hint is None else hint + mid_prompt = 'Question: ' + question + '\nContext: ' + context + '\nOptions: ' + options_prompt + ans_prompt = '[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is' + prompt = img_prompt + mid_prompt + ans_prompt + else: + mid_prompt = f'Answer the question using a single word or phrase.{question}' + ans_prompt = '[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + prompt = img_prompt + mid_prompt + ans_prompt + + return prompt + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + if DATASET_TYPE(dataset) == 'multi-choice': + prompt = self.build_mcqa(line) + elif DATASET_TYPE(dataset) == 'VQA': + if 'mathvista' in dataset.lower(): + q = line['question'] + prompt = f'[UNUSED_TOKEN_146]user\n{q}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + else: + q = line['question'] + prompt = ( + f'[UNUSED_TOKEN_146]user\nAnswer the question using a single word or phrase.{q}' + '[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + ) + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + return message diff --git a/vlmeval/vlm/xcomposer/xcomposer2_4KHD.py b/vlmeval/vlm/xcomposer/xcomposer2_4KHD.py new file mode 100644 index 0000000..4af554a --- /dev/null +++ b/vlmeval/vlm/xcomposer/xcomposer2_4KHD.py @@ -0,0 +1,235 @@ +import torch +from transformers import AutoModel, AutoTokenizer +from PIL import Image +from ..base import BaseModel +from ...smp import * +from ...utils import DATASET_TYPE +import numpy as np +import torchvision.transforms as transforms + +import re +pattern = re.compile(r'[A-Z]') + + +def padding_336(b): + width, height = b.size + tar = int(np.ceil(height / 336) * 336) + top_padding = int((tar - height) / 2) + bottom_padding = tar - height - top_padding + left_padding = 0 + right_padding = 0 + b = transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255, 255, 255]) + + return b + + +def HD_transform(img, im_num=16): + width, height = img.size + trans = False + if width < height: + img = img.transpose(Image.TRANSPOSE) + trans = True + width, height = img.size + ratio = (width / height) + scale = 1 + while scale * np.ceil(scale / ratio) <= im_num: + scale += 1 + scale -= 1 + new_w = int(scale * 336) + new_h = int(new_w / ratio) + + img = transforms.functional.resize(img, [new_h, new_w],) + img = padding_336(img) + width, height = img.size + assert width * height <= im_num * 336 * 336 + if trans: + img = img.transpose(Image.TRANSPOSE) + + return img + + +meta_instruction = """You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔). +- InternLM-XComposer (浦语·灵笔) is a multi-modality conversational language model that is developed\ + by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless. +- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen by\ + the user such as English and 中文. +- InternLM-XComposer (浦语·灵笔) is capable of comprehending and articulating responses\ + effectively based on the provided image.""" + + +def model_gen(model, text, images, need_bos=True, padding=False, beams=3, max_token=500): + pt1 = 0 + embeds = [] + im_mask = [] + images = [images] + images_loc = [0] + for i, pts in enumerate(images_loc + [len(text)]): + subtext = text[pt1:pts] + if need_bos or len(subtext) > 0: + text_embeds = model.encode_text(subtext, add_special_tokens=need_bos) + embeds.append(text_embeds) + im_mask.append(torch.zeros(text_embeds.shape[:2]).cuda()) + need_bos = False + if i < len(images): + try: + image = Image.open(images[i]).convert('RGB') + except: + image = images[i].convert('RGB') + + image = HD_transform(image, im_num=model.hd_num) + image = model.vis_processor(image).unsqueeze(0).cuda() + image_embeds = model.encode_img(image) + embeds.append(image_embeds) + im_mask.append(torch.ones(image_embeds.shape[:2]).cuda()) + pt1 = pts + embeds = torch.cat(embeds, dim=1) + im_mask = torch.cat(im_mask, dim=1) + im_mask = im_mask.bool() + + outputs = model.generate(inputs_embeds=embeds, im_mask=im_mask, + temperature=1.0, max_new_tokens=max_token, num_beams=beams, + do_sample=False, repetition_penalty=1.0) + output_token = outputs[0] + if output_token[0] == 0 or output_token[0] == 1: + output_token = output_token[1:] + output_text = model.tokenizer.decode(output_token, add_special_tokens=False) + output_text = output_text.split('[UNUSED_TOKEN_145]')[0].strip() + return output_text + + +class XComposer2_4KHD(BaseModel): + + INSTALL_REQ = False + INTERLEAVE = False + + def __init__(self, model_path='internlm/internlm-xcomposer2-4khd-7b', **kwargs): + assert model_path is not None + self.model_path = model_path + + model = AutoModel.from_pretrained(self.model_path, device_map='cpu', trust_remote_code=True).cuda().eval() + model.half() + tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True) + model.tokenizer = tokenizer + self.model = model + self.device = self.model.model.tok_embeddings.weight.device + self.model.hd_num = 25 + + def generate_mme(self, image_path, text): + text = text.split('Please answer')[0].strip() + text = f'{text} Answer this question briefly' + text = f'[UNUSED_TOKEN_146]user\n{text}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + + return model_gen(self.model, text, image_path, need_bos=True, padding=True, beams=5) + + def generate_multichoice(self, image_path, text, dataset): + out = model_gen(self.model, text, image_path, need_bos=True, padding=False, beams=5, max_token=5) + if 'mmmu' in dataset.lower(): + return out + res = pattern.findall(out) + if len(res) == 0: + print('Error:', out) + res = 'Z' + return res[0] + + def generate_vqa(self, image_path, text): + out = model_gen(self.model, text, image_path, need_bos=True, max_token=100) + return out + + def generate_vanilla(self, image_path, text): + out = model_gen(self.model, text, image_path, need_bos=True, max_token=500) + return out + + def generate_brief(self, image_path, text): + text = '[UNUSED_TOKEN_146]user\nAnswer the question using a single word or phrase.{}\ + [UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n'.format(text) + out = model_gen(self.model, text, image_path, need_bos=True, max_token=10) + return out + + def generate(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + if listinstr(['docvqa_test', 'infovqa_test'], dataset.lower()): + self.model.hd_num = 65 + elif listinstr(['docvqa_val', 'infovqa_val', 'OCRBench'], dataset.lower()): + self.model.hd_num = 55 + elif listinstr(['mmmu', 'mmbench', 'mmvet'], dataset.lower()): + self.model.hd_num = 16 + else: + self.model.hd_num = 25 + + with torch.cuda.amp.autocast(): + if dataset is None: + return self.generate_vanilla(image_path, prompt) + assert isinstance(dataset, str) + if dataset == 'MME': + return self.generate_mme(image_path, prompt) + + elif listinstr(['hallu'], dataset.lower()): + return self.generate_brief(image_path, prompt) + + elif listinstr(['llava', 'mmvet'], dataset.lower()): + return self.generate_vanilla(image_path, prompt) + + elif dataset is not None and DATASET_TYPE(dataset) == 'multi-choice': + return self.generate_multichoice(image_path, prompt, dataset) + + elif dataset is not None and DATASET_TYPE(dataset) == 'VQA': + return self.generate_vqa(image_path, prompt) + + else: + return self.generate_vanilla(image_path, prompt) + + def use_custom_prompt(self, dataset): + assert dataset is not None + if DATASET_TYPE(dataset) == 'multi-choice' or DATASET_TYPE(dataset) == 'VQA': + return True + return False + + def build_mcqa(self, line): + question = line['question'] + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + img_prompt = '[UNUSED_TOKEN_146]user\n' + if len(options): + options_prompt = '' + for key, item in options.items(): + options_prompt += f'{key}. {item} ' + options_prompt = options_prompt.strip() + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + + context = 'N/A' if hint is None else hint + mid_prompt = 'Question: ' + question + '\nContext: ' + context + '\nOptions: ' + options_prompt + ans_prompt = '[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is' + prompt = img_prompt + mid_prompt + ans_prompt + else: + mid_prompt = f'Answer the question using a single word or phrase.{question}' + ans_prompt = '[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + prompt = img_prompt + mid_prompt + ans_prompt + + return prompt + + def build_prompt(self, line, dataset=None): + assert dataset is None or isinstance(dataset, str) + assert self.use_custom_prompt(dataset) + tgt_path = self.dump_image(line, dataset) + + if DATASET_TYPE(dataset) == 'multi-choice': + prompt = self.build_mcqa(line) + elif DATASET_TYPE(dataset) == 'VQA': + if 'mathvista' in dataset.lower(): + q = line['question'] + prompt = f'[UNUSED_TOKEN_146]user\n{q}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + elif listinstr(['llava', 'mmvet'], dataset.lower()): + q = line['question'] + prompt = '[UNUSED_TOKEN_146]system\n{}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]user\n{}\ + Answer this question in detail.[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]\ + assistant\n'.format(meta_instruction, q) + else: + q = line['question'] + prompt = f'[UNUSED_TOKEN_146]user\nAnswer the question using a single word or phrase.\ + {q}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n' + ret = [dict(type='text', value=prompt)] + ret.extend([dict(type='image', value=s) for s in tgt_path]) + return ret diff --git a/vlmeval/vlm/yi_vl.py b/vlmeval/vlm/yi_vl.py new file mode 100644 index 0000000..86568a0 --- /dev/null +++ b/vlmeval/vlm/yi_vl.py @@ -0,0 +1,156 @@ +import torch +import sys +import os.path as osp +import warnings +from PIL import Image +from vlmeval.smp import get_cache_path, load, dump, splitlen +from huggingface_hub import snapshot_download +from .base import BaseModel + + +""" +You can perform inference of Yi-VL through the following steps: +1. clone the repo https://github.com/01-ai/Yi to path-to-Yi +2. set up the environment and install the required packages in path-to-Yi/VL/requirements.txt +3. set Yi_ROOT in vlmeval/config.py + Yi_ROOT = path-to-Yi + +You are all set now! To run a demo for Yi-VL: +```python +from vlmeval import * +model = supported_VLM['Yi_VL_6B']() +model.generate('apple.jpg', 'What is in this image?') +``` +To run evaluation for Yi-VL, use `python run.py --model Yi_VL_6B --data {dataset_list}` +""" + + +def edit_config(repo_id): + if not osp.exists(repo_id): + root = get_cache_path(repo_id) + else: + root = repo_id + assert root is not None and osp.exists(root) + cfg = osp.join(root, 'config.json') + data = load(cfg) + mm_vision_tower = data['mm_vision_tower'] + if mm_vision_tower.startswith('./vit/'): + data['mm_vision_tower'] = osp.join(root, mm_vision_tower) + assert osp.exists(data['mm_vision_tower']) + dump(data, cfg) + + +def disable_torch_init(): + """ + Disable the redundant torch default initialization to accelerate model creation. + """ + import torch + + setattr(torch.nn.Linear, 'reset_parameters', lambda self: None) + setattr(torch.nn.LayerNorm, 'reset_parameters', lambda self: None) + + +class Yi_VL(BaseModel): + + INSTALL_REQ = True + INTERLEAVE = False + + def __init__(self, + model_path='01-ai/Yi-VL-6B', + root=None, + **kwargs): + + if root is None: + warnings.warn( + 'Please set root to the directory of Yi, ' + 'which is cloned from here: https://github.com/01-ai/Yi.' + ) + + self.root = osp.join(root, 'VL') + sys.path.append(self.root) + + if splitlen(model_path, '/') == 2 and not osp.exists(model_path): + if get_cache_path(model_path) is None: + snapshot_download(repo_id=model_path) + edit_config(model_path) + elif osp.exists(model_path): + edit_config(model_path) + + from llava.mm_utils import get_model_name_from_path, load_pretrained_model + from llava.model.constants import key_info + + disable_torch_init() + key_info['model_path'] = model_path + get_model_name_from_path(model_path) + self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model( + model_path, + device_map='cpu') + self.model = self.model.cuda() + self.conv_mode = 'mm_default' + + kwargs_default = dict(temperature=0.2, + num_beams=1, + do_sample=False, + max_new_tokens=1024, + top_p=None) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def generate_inner(self, message, dataset=None): + prompt, image_path = self.message_to_promptimg(message) + + from llava.conversation import conv_templates + from llava.model.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX + from llava.mm_utils import KeywordsStoppingCriteria, expand2square, tokenizer_image_token + + qs = DEFAULT_IMAGE_TOKEN + '\n' + prompt + conv = conv_templates[self.conv_mode].copy() + conv.append_message(conv.roles[0], qs) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + input_ids = ( + tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt') + .unsqueeze(0) + .cuda() + ) + + image = Image.open(image_path) + if getattr(self.model.config, 'image_aspect_ratio', None) == 'pad': + if image.mode == 'L': + background_color = int(sum([int(x * 255) for x in self.image_processor.image_mean]) / 3) + else: + background_color = tuple(int(x * 255) for x in self.image_processor.image_mean) + image = expand2square(image, background_color) + image_tensor = self.image_processor.preprocess(image, return_tensors='pt')[ + 'pixel_values' + ][0] + + stop_str = conv.sep + keywords = [stop_str] + stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids) + self.model = self.model.to(dtype=torch.bfloat16) + with torch.inference_mode(): + output_ids = self.model.generate( + input_ids, + images=image_tensor.unsqueeze(0).to(dtype=torch.bfloat16).cuda(), + stopping_criteria=[stopping_criteria], + use_cache=True, + **self.kwargs) + + input_token_len = input_ids.shape[1] + n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item() + if n_diff_input_output > 0: + print( + f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids' + ) + outputs = self.tokenizer.batch_decode( + output_ids[:, input_token_len:], skip_special_tokens=True + )[0] + outputs = outputs.strip() + + if outputs.endswith(stop_str): + outputs = outputs[: -len(stop_str)] + outputs = outputs.strip() + return outputs