[Fix] Fix cache_dir issue where MVBench cannot be found (#306)

* [add] add internvideo2 support && change mvbench to video branch

* [add] answer_prompt of internvideo2

* [add] change video type of internvideo2

* [fix] update template of mvbench

* [reformat]

* [fix] generate_until_multi_round

* [Feat] videochat2 support

* [feat] Link cache_path to cache_dir if no unzip or untar

* [feat] new variable in dataset kwargs: create_link

---------

Co-authored-by: heyinan <[email protected]>

Author: yinanhe

lmms_eval/api/task.py

@@ -938,6 +938,7 @@ def _download_from_youtube(path):
                     force_download = dataset_kwargs.get("force_download", False)
                     force_unzip = dataset_kwargs.get("force_unzip", False)
                     revision = dataset_kwargs.get("revision", "main")
+                    create_link = dataset_kwargs.get("create_link", False)
                     cache_path = snapshot_download(repo_id=self.DATASET_PATH, revision=revision, repo_type="dataset", force_download=force_download, etag_timeout=60)
                     zip_files = glob(os.path.join(cache_path, "**/*.zip"), recursive=True)
                     tar_files = glob(os.path.join(cache_path, "**/*.tar*"), recursive=True)
@@ -1001,6 +1002,16 @@ def concat_tar_parts(tar_parts, output_tar):
                             if not os.path.exists(os.path.join(cache_dir, os.path.basename(base_name))):
                                 untar_video_data(output_tar)
 
+                    # Link cache_path to cache_dir if needed.
+                    if create_link:
+                        if not os.path.exists(cache_dir) or os.path.islink(cache_dir):
+                            if os.path.islink(cache_dir):
+                                os.remove(cache_dir)
+                                eval_logger.info(f"Removed existing symbolic link: {cache_dir}")
+                            # Create a new symbolic link
+                            os.symlink(cache_path, cache_dir)
+                            eval_logger.info(f"Symbolic link created successfully: {cache_path} -> {cache_dir}")
+
                 accelerator.wait_for_everyone()
                 dataset_kwargs.pop("cache_dir")
                 dataset_kwargs.pop("video")
@@ -1019,6 +1030,9 @@ def concat_tar_parts(tar_parts, output_tar):
             if "local_files_only" in dataset_kwargs:
                 dataset_kwargs.pop("local_files_only")
 
+            if "create_link" in dataset_kwargs:
+                dataset_kwargs.pop("create_link")
+
         self.dataset = datasets.load_dataset(
             path=self.DATASET_PATH,
             name=self.DATASET_NAME,

lmms_eval/tasks/mvbench/_default_template_yaml

@@ -4,6 +4,7 @@ dataset_kwargs:
   cache_dir: mvbench_video
   video: True
   revision: video
+  create_link: True
 generation_kwargs:
   max_new_tokens: 16
   temperature: 0