first Instruct Rephrasing approach; results 😶

SEED-Bench/InstructBlip_eval_rephrasing.py

@@ -76,6 +76,7 @@ def main():
 # .*\\n1\W+(.*)\W+\\n2\W+(.*)\W+\\n.*3\W+(.*)\W+\\n4\W+(.*)\W+\\n
 # regex for splitting up the captions from the new prompts
 
+
 if __name__ == '__main__':
     main()
     wandb.finish()

SEED-Bench/evaluator_strategies/BLIP2Models.py

@@ -13,7 +13,7 @@
 from transformers import Blip2ForConditionalGeneration, Blip2Processor, InstructBlipPreTrainedModel,InstructBlipForConditionalGeneration,InstructBlipProcessor
 import pandas as pd
 import json
-image_dir = "/net/mraid11/export/data/idanta/SEED/SEED-Bench-image"
+image_dir = "/home/projects/shimon/idanta/SemOOD/SEED-Bench-image"
 # All of the below URLs are taken from, and most of the implementation are heavily inspired from the wonderful https://github.com/salesforce/BLIP repo.
 
 download_urls = {

SEED-Bench/evaluator_strategies/InstructBlipModels.py

@@ -1,5 +1,6 @@
 from evaluator_strategies.BLIP2Models import SEEDModelWrapper
 from transformers import InstructBlipForConditionalGeneration, InstructBlipProcessor
+import torch
 
 class InstructBlipModel(SEEDModelWrapper):
     def __init__(self, root_dir, device, names, variant="InstructBLIP"):
@@ -71,7 +72,6 @@ def answer(self, batched_captions: str, processed_imgs: dict, batched_questions:
 
 
 
-
 class InstructBlipAnswerByRephrasing(InstructBlipModel):
     """Using the rephrasing strategy for instructBlip model inputs.
     The reason there is a different rephrasing class for each model is the ability to define where
@@ -95,8 +95,8 @@ def answer(self, batched_captions: str, processed_imgs: dict, batched_questions:
         """        
         scores = torch.zeros(batch_size, 4)
         for b_ind in range(batch_size):
-            processed_question = f"""Question: {batched_questions[b_ind]}\nAnswer:"""
-            procecced_question = self.processor(text=processed_question,return_tensors="pt", padding="longest").to(self.device)
+            processed_question = batched_questions[b_ind]
+            procecced_question = self.processor(text=processed_question, return_tensors="pt", padding="longest").to(self.device)
             # use the builtin query tokens
             input_tokenized = procecced_question
             query_tokens = self.model.query_tokens.expand(batch_size, -1, -1)

scores_for_InstructBLIP_under_InstructBlipAnswerByRephrasing.csv

@@ -0,0 +1,96 @@
+,0,1,2,3
+0,4.607864,5.0134974,4.4213905,4.381195
+1,2.291056,2.1802828,2.3044882,2.5354338
+2,3.5580587,4.0757885,4.051968,3.4076622
+3,2.9630399,3.0302708,2.6874132,2.5891275
+4,2.9773328,3.0189648,3.047992,1.7474427
+5,2.9575756,2.5105,3.6802256,2.4340236
+6,2.119201,3.6195443,2.629217,2.0616457
+7,2.4508975,2.422275,2.031524,1.8537651
+8,3.6731234,2.7567835,2.5461571,2.317345
+9,2.2903724,2.1880815,2.3362374,2.582191
+10,2.512866,2.3766422,2.481413,2.8755012
+11,4.245689,4.0315075,3.5623407,3.3885863
+12,3.5502715,2.6717608,3.0829134,2.4075463
+13,1.9999863,2.1492288,1.9471977,1.3450829
+14,2.6231525,2.5338597,2.3850787,2.0335345
+15,3.699109,3.9398804,4.0545855,2.4351296
+16,3.4837837,2.7989333,3.7956488,1.9452451
+17,3.2603753,3.7857354,3.5546958,2.3361242
+18,2.6479676,2.5837858,2.7224193,2.1264129
+19,3.0880423,2.761546,4.195385,2.4407084
+20,4.0170927,3.1780393,3.8511362,5.3393345
+21,5.1405644,4.259119,4.5953736,3.0673811
+22,2.2212954,2.040456,2.3317456,2.381661
+23,3.8256915,3.6523855,2.9521618,2.8250587
+24,3.0462246,2.8033395,3.7258008,4.068088
+25,2.5185814,2.910927,2.1492593,2.323902
+26,4.5851693,4.3782907,5.530996,1.6010021
+27,3.1346166,3.1554322,4.268629,2.457543
+28,3.933734,4.817355,4.5250535,4.7430706
+29,2.5486007,2.5833983,2.5485716,2.2981822
+30,3.5739536,4.2559595,3.2649412,2.3632455
+31,3.804528,3.7552752,3.0693371,2.9702425
+32,2.80277,2.8083103,2.8128455,2.067571
+33,3.3812268,2.7131045,2.3027399,2.0644073
+34,5.4046507,4.819594,5.4737034,4.022655
+35,2.9837337,2.4042144,2.942005,2.0829804
+36,2.8847072,2.9396806,3.4859712,1.857147
+37,3.0433702,3.0415866,3.4574687,2.3021274
+38,3.3750167,3.8209372,3.56107,2.415555
+39,2.2346926,2.8832994,4.390423,1.5179881
+40,1.9658923,2.4658682,2.5567808,1.5041803
+41,2.6658442,2.7604868,2.662494,1.8975233
+42,4.3486443,3.3310955,3.7894917,3.821133
+43,2.9390786,3.082584,3.422623,2.1132839
+44,3.2569125,3.1567938,3.131862,3.1150239
+45,2.5982165,2.8886,3.277272,2.3803298
+46,2.359545,2.4986327,2.3824725,2.0194385
+47,3.3263993,3.4367547,2.0880883,2.653087
+48,3.3007107,3.7436233,3.9460356,2.4938636
+49,2.800818,2.3732505,3.3671167,1.9799358
+50,4.756874,4.023073,4.476115,2.2328968
+51,2.8056366,2.8056366,3.4303195,2.0976
+52,2.2293866,2.445348,2.2981784,1.6814452
+53,3.7301621,2.9077306,2.5342214,2.4706354
+54,3.2484891,3.6295083,3.3155096,1.9830467
+55,1.8343823,2.257357,1.9441292,1.5333092
+56,3.2084217,3.394591,3.9378831,2.3361082
+57,2.711235,2.8090603,3.4687684,2.1011984
+58,3.0005426,3.452554,3.4766967,2.584944
+59,5.01485,2.6209836,3.0907714,2.2707007
+60,3.5804274,3.6917522,3.716403,2.897238
+61,1.8295434,2.4167907,2.6026149,1.6643047
+62,3.7862837,4.2137966,3.7097437,2.6154056
+63,3.0338967,2.7606127,3.476089,1.7573159
+64,2.7552476,2.531338,2.691341,1.9970194
+65,3.0138147,2.8544457,3.582118,1.3507941
+66,2.5973022,2.6308644,3.0829508,2.4337323
+67,2.4101453,2.9280293,2.9199502,1.7114403
+68,2.659663,2.557051,2.7225938,2.939801
+69,2.3896894,2.3896894,2.3896894,1.6901537
+70,3.0700908,3.1722798,2.4887345,2.0368843
+71,4.8222876,2.2780712,3.661358,3.8150568
+72,3.070705,3.4450226,4.010922,2.0032527
+73,3.3190246,2.8785248,3.4819117,2.073287
+74,3.1152065,2.9141529,2.6376088,2.1683576
+75,2.9289465,2.805173,3.9439306,3.4187567
+76,3.548537,2.9084914,3.366692,2.7352602
+77,2.686587,3.21833,2.4818308,2.2770114
+78,3.9359922,3.8318079,4.0355206,3.0276597
+79,1.8463275,2.8908932,3.0413854,2.9308794
+80,2.8049202,2.6734195,3.79492,1.83825
+81,2.6314633,3.2177527,2.673124,2.1066804
+82,3.0841231,2.8757508,2.5257063,2.1614788
+83,3.619032,3.4112535,2.7468803,2.3791053
+84,3.27178,3.5019367,2.8567715,2.2212155
+85,2.377861,2.6658874,2.7757494,1.678216
+86,3.687883,3.7816157,3.9250722,1.9565883
+87,3.4063253,2.3597436,3.7542894,2.4077
+88,2.8476057,2.852884,2.6279716,2.87723
+89,3.1102738,2.8074834,3.1797128,2.6968644
+90,1.4997077,1.4931053,1.433183,1.703876
+91,3.0672376,2.5909798,3.06019,1.9696704
+92,3.1011984,3.0995133,3.2578585,1.8669808
+93,2.6545327,2.552573,2.3295782,1.7995411
+94,5.013773,5.273729,5.2492776,3.4098427