getPearsonandRanking.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8d9af7d4-5502-4378-94e8-0f80050c24d2",
   "metadata": {},
   "source": [
    "## Importing required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92eacafb-dd26-4de1-85c1-5481b6dbbcc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import io, sys, os\n",
    "from functools import reduce\n",
    "import spacy\n",
    "import string\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from spacy.lang.en import English\n",
    "import numpy as np\n",
    "import random\n",
    "import re\n",
    "nlp = English()\n",
    "tokenizer = nlp.tokenizer\n",
    "from tqdm.notebook import tqdm\n",
    "import plotly.express as px\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from datasets import load_metric\n",
    "from torch import nn\n",
    "from scipy import stats\n",
    "from scipy.spatial.distance import euclidean\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c748b92e-b82c-471c-b61f-b93ec592f755",
   "metadata": {},
   "source": [
    "## Preprocessing all language pairs' data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42a5d405-0742-40dd-8ae7-3a2bb0719c62",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres_et = []\n",
    "dfdevet = pd.read_csv(\"et_en_data/dev.eten.df.short.tsv\",sep='\\t')\n",
    "dftestet = pd.read_csv(\"et_en_data/test20.eten.df.short.tsv\", sep='\\t')\n",
    "frameset = [dfdevet, dftestet]\n",
    "finalet = pd.concat(frameset)\n",
    "combet = finalet.reset_index(drop=True)\n",
    "devtestCombet = combet.drop(columns=['index', 'mean', 'z_scores', 'z_mean', 'model_scores', 'scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e234f57-1257-43da-96a1-756a7813df5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres_ru = []\n",
    "dfdevru = pd.read_csv(\"ru_en_data/dev.ruen.df.short.tsv\",sep='\\t')\n",
    "dftestru = pd.read_csv(\"ru_en_data/test20.ruen.df.short.tsv\", sep='\\t')\n",
    "framesru = [dfdevru, dftestru]\n",
    "finalru = pd.concat(framesru)\n",
    "combru = finalru.reset_index(drop=True)\n",
    "devtestCombru = combru.drop(columns=['segid', 'mean', 'z_scores', 'z_mean', 'model_scores', 'scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f81283c4-a6a8-468e-8069-5d9675ed32a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres_ro = []\n",
    "dfdevro = pd.read_csv(\"ro_en_data/dev.roen.df.short.tsv\",sep='\\t')\n",
    "dftestro = pd.read_csv(\"ro_en_data/test20.roen.df.short.tsv\", sep='\\t')\n",
    "framesro = [dfdevro, dftestro]\n",
    "finalro = pd.concat(framesro)\n",
    "combro = finalro.reset_index(drop=True)\n",
    "devtestCombro = combro.drop(columns=['index', 'mean', 'z_scores', 'z_mean', 'model_scores', 'scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01cedebe-bce7-4b68-ba4e-0b2fd72496cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres_si = []\n",
    "dfdevsi = pd.read_csv(\"si_en_data/dev.sien.df.short.tsv\",sep='\\t')\n",
    "dftestsi = pd.read_csv(\"si_en_data/test20.sien.df.short.tsv\", sep='\\t')\n",
    "framessi = [dfdevsi, dftestsi]\n",
    "finalsi = pd.concat(framessi)\n",
    "combsi = finalsi.reset_index(drop=True)\n",
    "devtestCombsi = combsi.drop(columns=['index', 'mean', 'z_scores', 'z_mean', 'model_scores', 'scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e162a914-b0d6-41ed-b6c7-1fb2b53d338d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres_ne = []\n",
    "dfdevne = pd.read_csv(\"ne_en_data/dev.neen.df.short.tsv\",sep='\\t')\n",
    "dftestne = pd.read_csv(\"ne_en_data/test20.neen.df.short.tsv\", sep='\\t')\n",
    "framesne = [dfdevne, dftestne]\n",
    "finalne = pd.concat(framesne)\n",
    "combne = finalne.reset_index(drop=True)\n",
    "devtestCombne = combne.drop(columns=['index', 'mean', 'z_scores', 'z_mean', 'model_scores', 'scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38116f4d-5f66-418d-8f93-50fb3465c117",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transquest.algo.sentence_level.siamesetransquest.run_model import SiameseTransQuestModel\n",
    "modelEt = SiameseTransQuestModel(\"TransQuest/siamesetransquest-da-et_en-wiki\")\n",
    "modelRu = SiameseTransQuestModel(\"TransQuest/siamesetransquest-da-ru_en-reddit_wikiquotes\")\n",
    "modelRo = SiameseTransQuestModel(\"TransQuest/siamesetransquest-da-ro_en-wiki\")\n",
    "modelSi = SiameseTransQuestModel(\"TransQuest/siamesetransquest-da-si_en-wiki\")\n",
    "modelNe = SiameseTransQuestModel(\"TransQuest/siamesetransquest-da-ne_en-wiki\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a400d089-66f1-4324-93cc-8a6f9998ef02",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtlolet = devtestCombet.values.tolist()\n",
    "nppredictionset = modelEt.predict(dtlolet) ## For Estonian\n",
    "df_orig_trans_scoreet = devtestCombet\n",
    "df_orig_trans_scoreet['trans_score_siam'] = nppredictionset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1da3979-7188-4d91-8dfa-a62558b4ea09",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtlolru = devtestCombru.values.tolist()\n",
    "nppredictionsru = modelRu.predict(dtlolru) ## For Russian\n",
    "df_orig_trans_scoreru = devtestCombru\n",
    "df_orig_trans_scoreru['trans_score_siam'] = nppredictionsru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f15628c7-2e4b-4906-9c1b-636ed5a02a56",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtlolro = devtestCombro.values.tolist()\n",
    "nppredictionsro = modelRo.predict(dtlolro) ## For Romanian\n",
    "df_orig_trans_scorero = devtestCombro\n",
    "df_orig_trans_scorero['trans_score_siam'] = nppredictionsro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f911ad3-a801-44be-a090-790b307ab5c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtlolsi = devtestCombsi.values.tolist()\n",
    "nppredictionssi = modelSi.predict(dtlolsi) ## For Sinhala\n",
    "df_orig_trans_scoresi = devtestCombsi\n",
    "df_orig_trans_scoresi['trans_score_siam'] = nppredictionssi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "516e3f51-dca8-4537-8cc3-bdc8b47d374f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtlolne = devtestCombne.values.tolist()\n",
    "nppredictionsne = modelNe.predict(dtlolne) ## For Nepali\n",
    "df_orig_trans_scorene = devtestCombne\n",
    "df_orig_trans_scorene['trans_score_siam'] = nppredictionsne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56161b08-a034-48d7-ad98-bc5b2ef63939",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = combet['scores'].tolist()\n",
    "newlist = []\n",
    "sublist =[]\n",
    "for sublist in l:\n",
    "    sublist = sublist.split(\", \")\n",
    "    sublist = [i.strip('[]') for i in sublist]\n",
    "    sublist = [float(i) for i in sublist]\n",
    "    newlist.append(sublist)\n",
    "\n",
    "newlistnparray = np.array(newlist)\n",
    "newlistmeans = np.mean(newlistnparray, axis=1)\n",
    "newlistmeans = newlistmeans/100\n",
    "newlistmeans = newlistmeans.tolist()\n",
    "df_orig_trans_scoreet['trans_score_hum'] = newlistmeans\n",
    "\n",
    "pearsonSiameseEt = stats.pearsonr(df_orig_trans_scoreet['trans_score_siam'].values.tolist(), df_orig_trans_scoreet['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dee8b2e-67a0-410d-b723-0febf5b6b2af",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = combru['scores'].tolist()\n",
    "newlist = []\n",
    "sublist =[]\n",
    "for sublist in l:\n",
    "    sublist = sublist.split(\", \")\n",
    "    sublist = [i.strip('[]') for i in sublist]\n",
    "    sublist = [float(i) for i in sublist]\n",
    "    newlist.append(sublist)\n",
    "\n",
    "newlistnparray = np.array(newlist)\n",
    "newlistmeans = np.mean(newlistnparray, axis=1)\n",
    "newlistmeans = newlistmeans/100\n",
    "newlistmeans = newlistmeans.tolist()\n",
    "df_orig_trans_scoreru['trans_score_hum'] = newlistmeans\n",
    "\n",
    "pearsonSiameseRu = stats.pearsonr(df_orig_trans_scoreru['trans_score_siam'].values.tolist(), df_orig_trans_scoreru['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af537cce-9b0e-48c1-832e-1f51b8be2dcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = combro['scores'].tolist()\n",
    "newlist = []\n",
    "sublist =[]\n",
    "for sublist in l:\n",
    "    sublist = sublist.split(\", \")\n",
    "    sublist = [i.strip('[]') for i in sublist]\n",
    "    sublist = [float(i) for i in sublist]\n",
    "    newlist.append(sublist)\n",
    "\n",
    "newlistnparray = np.array(newlist)\n",
    "newlistmeans = np.mean(newlistnparray, axis=1)\n",
    "newlistmeans = newlistmeans/100\n",
    "newlistmeans = newlistmeans.tolist()\n",
    "df_orig_trans_scorero['trans_score_hum'] = newlistmeans\n",
    "\n",
    "pearsonSiameseRo = stats.pearsonr(df_orig_trans_scorero['trans_score_siam'].values.tolist(), df_orig_trans_scorero['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d64adf37-c01a-4e5d-a4de-3d8064611599",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = combsi['scores'].tolist()\n",
    "newlist = []\n",
    "sublist =[]\n",
    "for sublist in l:\n",
    "    sublist = sublist.split(\", \")\n",
    "    sublist = [i.strip('[]') for i in sublist]\n",
    "    sublist = [float(i) for i in sublist]\n",
    "    newlist.append(sublist)\n",
    "\n",
    "newlistnparray = np.array(newlist)\n",
    "newlistmeans = np.mean(newlistnparray, axis=1)\n",
    "newlistmeans = newlistmeans/100\n",
    "newlistmeans = newlistmeans.tolist()\n",
    "df_orig_trans_scoresi['trans_score_hum'] = newlistmeans\n",
    "\n",
    "pearsonSiameseSi = stats.pearsonr(df_orig_trans_scoresi['trans_score_siam'].values.tolist(), df_orig_trans_scoresi['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdd1295d-bf23-4c6a-83de-ef36800868a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = combne['scores'].tolist()\n",
    "newlist = []\n",
    "sublist =[]\n",
    "for sublist in l:\n",
    "    sublist = sublist.split(\", \")\n",
    "    sublist = [i.strip('[]') for i in sublist]\n",
    "    sublist = [float(i) for i in sublist]\n",
    "    newlist.append(sublist)\n",
    "\n",
    "newlistnparray = np.array(newlist)\n",
    "newlistmeans = np.mean(newlistnparray, axis=1)\n",
    "newlistmeans = newlistmeans/100\n",
    "newlistmeans = newlistmeans.tolist()\n",
    "df_orig_trans_scorene['trans_score_hum'] = newlistmeans\n",
    "\n",
    "pearsonSiameseNe = stats.pearsonr(df_orig_trans_scorene['trans_score_siam'].values.tolist(), df_orig_trans_scorene['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a276964e-659d-4211-927d-cce77c3d0dac",
   "metadata": {},
   "source": [
    "### Pearson Correlations between human judgements and scores obtained from the SiameseTransQuest model(s). (Table 5 from the paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5633bf01-320b-45b0-9bc6-de697436f0c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonSiameseEt[0], pearsonSiameseRu[0], pearsonSiameseRo[0], pearsonSiameseSi[0], pearsonSiameseNe[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fa02253-a7a5-4705-862a-cabe27bc10c1",
   "metadata": {},
   "source": [
    "### USE THIS CODE BLOCK TO CLEAR THE GPU MEMORY IN CASE YOU DO NOT HAVE ENOUGH GPU MEMORY FOR LOADING ALL FIVE MODELS AT A TIME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdb3479c-d13a-40f9-b9a9-6f53126ce36e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install numba\n",
    "# from numba import cuda\n",
    "# cuda.select_device(0)\n",
    "# cuda.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21ae2141-1297-4cbf-8bec-76f4b0b48cbc",
   "metadata": {},
   "source": [
    "### Loading all MonoTransQuest models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0bf842c-5ca8-43ab-984b-565e366d3b8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transquest.algo.sentence_level.monotransquest.run_model import MonoTransQuestModel\n",
    "\n",
    "modelEt = MonoTransQuestModel(\"xlmroberta\", \"TransQuest/monotransquest-da-et_en-wiki\", num_labels=1, use_cuda=torch.cuda.is_available())\n",
    "modelRu = MonoTransQuestModel(\"xlmroberta\", \"TransQuest/monotransquest-da-ru_en-reddit_wikiquotes\", num_labels=1, use_cuda=torch.cuda.is_available())\n",
    "modelRo = MonoTransQuestModel(\"xlmroberta\", \"TransQuest/monotransquest-da-ro_en-wiki\", num_labels=1, use_cuda=torch.cuda.is_available())\n",
    "modelSi = MonoTransQuestModel(\"xlmroberta\", \"TransQuest/monotransquest-da-si_en-wiki\", num_labels=1, use_cuda=torch.cuda.is_available())\n",
    "modelNe = MonoTransQuestModel(\"xlmroberta\", \"TransQuest/monotransquest-da-ne_en-wiki\", num_labels=1, use_cuda=torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fa7c983-3d69-42aa-b08f-fac4b41a3505",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionset, raw = modelEt.predict(dtlolet) \n",
    "df_orig_trans_scoreet = devtestCombet\n",
    "df_orig_trans_scoreet['trans_score_mono'] = nppredictionset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0784441e-3da2-467b-8278-d431f7cb9056",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionsru, raw = modelRu.predict(dtlolru) \n",
    "df_orig_trans_scoreru = devtestCombru\n",
    "df_orig_trans_scoreru['trans_score_mono'] = nppredictionsru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4f2023e-5dc0-4269-b612-205eb276b8f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionsro, raw = modelRo.predict(dtlolro) \n",
    "df_orig_trans_scorero = devtestCombro\n",
    "df_orig_trans_scorero['trans_score_mono'] = nppredictionsro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b8690e8-42ac-43db-ae87-1310fff7be24",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionssi, raw = modelRo.predict(dtlolsi)\n",
    "df_orig_trans_scoresi = devtestCombsi\n",
    "df_orig_trans_scoresi['trans_score_mono'] = nppredictionssi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27b08ad2-0707-4488-9c79-fd4891936de5",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionsne, raw = modelNe.predict(dtlolne)\n",
    "df_orig_trans_scorene = devtestCombne\n",
    "df_orig_trans_scorene['trans_score_mono'] = nppredictionsne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31e6f217-ca4f-4ae4-a177-8910b85f694b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonMonoEt = stats.pearsonr(df_orig_trans_scoreet['trans_score_mono'].values.tolist(), df_orig_trans_scoreet['trans_score_hum'].values.tolist())\n",
    "pearsonMonoRu = stats.pearsonr(df_orig_trans_scoreru['trans_score_mono'].values.tolist(), df_orig_trans_scoreru['trans_score_hum'].values.tolist())\n",
    "pearsonMonoRo = stats.pearsonr(df_orig_trans_scorero['trans_score_mono'].values.tolist(), df_orig_trans_scorero['trans_score_hum'].values.tolist())\n",
    "pearsonMonoSi = stats.pearsonr(df_orig_trans_scoresi['trans_score_mono'].values.tolist(), df_orig_trans_scoresi['trans_score_hum'].values.tolist())\n",
    "pearsonMonoNe = stats.pearsonr(df_orig_trans_scorene['trans_score_mono'].values.tolist(), df_orig_trans_scorene['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f11ec540-7148-4504-b315-219e7409a787",
   "metadata": {},
   "source": [
    "### Pearson Correlations between human judgements and scores obtained from the MonoTransQuest model(s). (Table 5 from the paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bc28c9e-0246-4938-be4a-d725efb8b45c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonMonoEt[0], pearsonMonoRu[0], pearsonMonoRo[0], pearsonMonoSi[0], pearsonMonoNe[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4af3319-a581-4ac1-9b7d-c9db32b28738",
   "metadata": {},
   "source": [
    "### Loading the MultiTransQuest model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cb404a8-65bd-44f4-b531-98f7721c8a17",
   "metadata": {},
   "outputs": [],
   "source": [
    "modelMulti = MonoTransQuestModel(\"xlmroberta\", \"TransQuest/monotransquest-da-multilingual\", num_labels=1, use_cuda=torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "426d4062-c92a-4804-a278-e495458bed29",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionset, raw = modelMulti.predict(dtlolet) \n",
    "df_orig_trans_scoreet = devtestCombet\n",
    "df_orig_trans_scoreet['trans_score_multi'] = nppredictionset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dcf9f42-cb8a-4b17-973a-a4619f946019",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionsru, raw = modelMulti.predict(dtlolru)\n",
    "df_orig_trans_scoreru = devtestCombru\n",
    "df_orig_trans_scoreru['trans_score_multi'] = nppredictionsru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45ab49c2-a62f-409d-9341-55f1294104f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionsro, raw = modelMulti.predict(dtlolro) \n",
    "df_orig_trans_scorero = devtestCombro\n",
    "df_orig_trans_scorero['trans_score_multi'] = nppredictionsro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bbcfe55-bb58-4242-bf58-92a8449420b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionssi, raw = modelMulti.predict(dtlolsi)\n",
    "df_orig_trans_scoresi = devtestCombsi\n",
    "df_orig_trans_scoresi['trans_score_multi'] = nppredictionssi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "753e9953-842e-433d-a4f9-70bdd44153a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "nppredictionsne, raw = modelMulti.predict(dtlolne)\n",
    "df_orig_trans_scorene = devtestCombne\n",
    "df_orig_trans_scorene['trans_score_multi'] = nppredictionsne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "204c8b3e-3104-451c-82f4-a72f5f0430b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonMultiEt = stats.pearsonr(df_orig_trans_scoreet['trans_score_multi'].values.tolist(), df_orig_trans_scoreet['trans_score_hum'].values.tolist())\n",
    "pearsonMultiRu = stats.pearsonr(df_orig_trans_scoreru['trans_score_multi'].values.tolist(), df_orig_trans_scoreru['trans_score_hum'].values.tolist())\n",
    "pearsonMultiRo = stats.pearsonr(df_orig_trans_scorero['trans_score_multi'].values.tolist(), df_orig_trans_scorero['trans_score_hum'].values.tolist())\n",
    "pearsonMultiSi = stats.pearsonr(df_orig_trans_scoresi['trans_score_multi'].values.tolist(), df_orig_trans_scoresi['trans_score_hum'].values.tolist())\n",
    "pearsonMultiNe = stats.pearsonr(df_orig_trans_scorene['trans_score_multi'].values.tolist(), df_orig_trans_scorene['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c8e1cd8-7bca-490e-b46f-406cdb75ecd1",
   "metadata": {},
   "source": [
    "### Pearson Correlations between human judgements and scores obtained from the Multilingual TransQuest model. (Table 5 from the paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "209d08b7-5966-47c2-a72f-8f8bb2b74ea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonMultiEt[0], pearsonMultiRu[0], pearsonMultiRo[0], pearsonMultiSi[0], pearsonMultiNe[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7485ff9d-7fde-454d-81f7-edb0bbcd7ec9",
   "metadata": {},
   "source": [
    "### Unsupervised method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e3607d6-845e-4cd6-a241-b8c13d569a25",
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_metrics(*args, **kwargs):\n",
    "    assert len(args) == len(kwargs[\"corr\"])\n",
    "    assert len(args[0]) == len(args[1])\n",
    "    \n",
    "    output = []\n",
    "    \n",
    "    for i in range(len(args[0])):\n",
    "        value = 0\n",
    "        for sign, metric in zip(kwargs[\"corr\"],args):\n",
    "            if sign > 0:\n",
    "                value += np.exp(metric[i])\n",
    "            else:\n",
    "                value += np.exp(1-metric[i])\n",
    "        output.append(value)\n",
    "    return output\n",
    "\n",
    "bert_score_metric = load_metric('bertscore', keep_in_memory=False, cache_dir=sys.path[0])\n",
    "bert_score_model = 'xlm-roberta-base'\n",
    "cos_sim_model = SentenceTransformer('xlm-r-bert-base-nli-stsb-mean-tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "158da28f-36bb-4c5e-9c0e-ccbc1626326d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_scores_orig(df, nTrials=1):\n",
    "    ## Need a temporary scoring function here. \n",
    "    scores = np.zeros((1, df.shape[0]))\n",
    "  \n",
    "    for i in tqdm(range(nTrials)):\n",
    "        embed_sent1 = cos_sim_model.encode(df['translation'].values.tolist(), convert_to_tensor=True)\n",
    "        embed_sent2 = cos_sim_model.encode(df['original'].values.tolist(), convert_to_tensor=True)\n",
    "        cos_sim = nn.CosineSimilarity(dim=1)(embed_sent1,embed_sent2)\n",
    "        cos_sim = (cos_sim -torch.min(cos_sim))/ (torch.max(cos_sim)-torch.min(cos_sim))\n",
    "        cos_scores = cos_sim.cpu().numpy()\n",
    "\n",
    "        bert_score_metric.add_batch(predictions=df['translation'].values.tolist(), references=df['original'].values.tolist())\n",
    "        score = bert_score_metric.compute(model_type=bert_score_model)\n",
    "        norm_score = (score[\"f1\"] - np.min(score[\"f1\"])) / (np.max(score[\"f1\"]) - np.min(score[\"f1\"]))\n",
    "        compound_metric = combine_metrics(cos_scores, norm_score, corr=[1, 1])\n",
    "        df_norm = pd.DataFrame()\n",
    "        df_norm['scores'] = compound_metric\n",
    "        df_norm['scores'] = (df_norm['scores'] - df_norm['scores'].min()) / (df_norm['scores'].max() - df_norm['scores'].min())\n",
    "        predictions = df_norm['scores'].values.tolist()\n",
    "    \n",
    "        scores[i] = predictions\n",
    "\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e68f635-62aa-4c3a-9c5d-1efff4c6e542",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_orig=[]\n",
    "scores_orig = predict_scores_orig(df_orig_trans_scoreet)\n",
    "df_orig_trans_scoreet['trans_score_unsupervised'] = scores_orig[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9091e6c4-cae0-4f68-80f5-fee46bfb9b75",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_orig=[]\n",
    "scores_orig = predict_scores_orig(df_orig_trans_scoreru)\n",
    "df_orig_trans_scoreru['trans_score_unsupervised'] = scores_orig[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef9d5b61-507d-421a-8db4-9c2c342adb69",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_orig=[]\n",
    "scores_orig = predict_scores_orig(df_orig_trans_scorero)\n",
    "df_orig_trans_scorero['trans_score_unsupervised'] = scores_orig[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ea355ac-b578-49cf-89ba-cf3483ddfaf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_orig=[]\n",
    "scores_orig = predict_scores_orig(df_orig_trans_scoresi)\n",
    "df_orig_trans_scoresi['trans_score_unsupervised'] = scores_orig[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9ed2c89-a87d-4b31-bab5-28181a938f15",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_orig=[]\n",
    "scores_orig = predict_scores_orig(df_orig_trans_scorene)\n",
    "df_orig_trans_scorene['trans_score_unsupervised'] = scores_orig[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4942dae-13eb-4f68-890d-0b92f47d447f",
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonUnsupervisedEt = stats.pearsonr(df_orig_trans_scoreet['trans_score_unsupervised'].values.tolist(), df_orig_trans_scoreet['trans_score_hum'].values.tolist())\n",
    "pearsonUnsupervisedRu = stats.pearsonr(df_orig_trans_scoreru['trans_score_unsupervised'].values.tolist(), df_orig_trans_scoreru['trans_score_hum'].values.tolist())\n",
    "pearsonUnsupervisedRo = stats.pearsonr(df_orig_trans_scorero['trans_score_unsupervised'].values.tolist(), df_orig_trans_scorero['trans_score_hum'].values.tolist())\n",
    "pearsonUnsupervisedSi = stats.pearsonr(df_orig_trans_scoresi['trans_score_unsupervised'].values.tolist(), df_orig_trans_scoresi['trans_score_hum'].values.tolist())\n",
    "pearsonUnsupervisedNe = stats.pearsonr(df_orig_trans_scorene['trans_score_unsupervised'].values.tolist(), df_orig_trans_scorene['trans_score_hum'].values.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ec8614c-3c29-44b3-b1c9-31e87f281c16",
   "metadata": {},
   "source": [
    "### Pearson Correlations between human judgements and scores obtained from the Unsupervised method. (Table 5 from the paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f25a4ca-1ddb-4a64-8bd8-337bd5b22a1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonUnsupervisedEt[0], pearsonUnsupervisedRu[0], pearsonUnsupervisedRo[0], pearsonUnsupervisedSi[0], pearsonUnsupervisedNe[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2ccc732-3c6a-4ba9-92a3-8098997f22eb",
   "metadata": {},
   "source": [
    "## This cell below has precomputed values from our original experiment.\n",
    "### Uncomment the code below to generate system rankings as obtained with the original experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "707e813b-796a-4fe8-912b-2413c184d205",
   "metadata": {},
   "outputs": [],
   "source": [
    "# etRanking = [-0.07716989078923159, -0.0003569826549955548, -0.04823801347576072, -0.011562646249216116, 0.009073152397363508]\n",
    "# ruRanking = [-0.12114444232591343, -0.007808453126371578, -0.05595449589693772, 0.007777049013473376, -0.03316327903357645]\n",
    "# roRanking = [-0.08864646480583055, -0.046775697881407216, -0.06406304652518169, -0.032233587293031205, 0.016345227061293727]\n",
    "# siRanking = [-0.05806420791682877, -0.020244334000706014, -0.04098220828783415, -0.02550455062267074, -0.0550465349628457]\n",
    "# neRanking = [-0.07638170781723766, -0.028229749720718452, -0.05293691062280626, -0.017175247487624883, -0.03316327921984096]\n",
    "\n",
    "# rows = [\"MonoTransQuest\", \"SiameseTransQuest\", \"MultiTransQuest\", \"Unsupervised\", \"OpenKiwi\"]\n",
    "\n",
    "# plt.figure(figsize=(9, 5))\n",
    "# plt.bar(rows, etRanking)\n",
    "# plt.suptitle('Et-En Ranked Systems')\n",
    "# plt.show()\n",
    "# plt.figure(figsize=(9, 5))\n",
    "# plt.bar(rows, ruRanking)\n",
    "# plt.suptitle('Ru-En Ranked Systems')\n",
    "# plt.show()\n",
    "# plt.figure(figsize=(9, 5))\n",
    "# plt.bar(rows, siRanking)\n",
    "# plt.suptitle('Si-En Ranked Systems')\n",
    "# plt.show()\n",
    "# plt.figure(figsize=(9, 5))\n",
    "# plt.bar(rows, roRanking)\n",
    "# plt.suptitle('Ro-En Ranked Systems')\n",
    "# plt.show()\n",
    "# plt.figure(figsize=(9, 5))\n",
    "# plt.bar(rows, neRanking)\n",
    "# plt.suptitle('Ne-En Ranked Systems')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cd1d0a4-18b4-40ba-aefd-06e6dc537c14",
   "metadata": {},
   "source": [
    "### If you further wish to compute the system rankings and are replicating our experiments, before moving further with the code, \n",
    "### please ensure that you have generated mean and variance values for all four methods, and for all language pairs.\n",
    "### You should save all these values (saving code already provided at the end of each notebook) in the NPY files.\n",
    "\n",
    "### The cell below shows how to recompute them from the dumped means values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6da18ba3-fa06-44b4-a59c-55e6b487358c",
   "metadata": {},
   "outputs": [],
   "source": [
    "etMeans_mono = np.load(\"commonResults/etMeans_mono.npy\")\n",
    "\n",
    "etMPPAvg_mono = (etMeans_mono[0] + etMeans_mono[1] + etMeans_mono[2] + etMeans_mono[3] + etMeans_mono[4] + etMeans_mono[5])/6\n",
    "etMAPAvg_mono = (etMeans_mono[6] + etMeans_mono[7] + etMeans_mono[8] + etMeans_mono[9] + etMeans_mono[10] + etMeans_mono[11] + etMeans_mono[12] + etMeans_mono[13])/8\n",
    "\n",
    "etDiff_mono = etMPPAvg_mono - etMAPAvg_mono\n",
    "\n",
    "ruMeans_mono = np.load(\"commonResults/ruMeans_mono.npy\")\n",
    "\n",
    "ruMPPAvg_mono = (ruMeans_mono[0] + ruMeans_mono[1] + ruMeans_mono[2] + ruMeans_mono[3] + ruMeans_mono[4] + ruMeans_mono[5])/6\n",
    "ruMAPAvg_mono = (ruMeans_mono[6] + ruMeans_mono[7] + ruMeans_mono[8] + ruMeans_mono[9] + ruMeans_mono[10] + ruMeans_mono[11] + ruMeans_mono[12] + ruMeans_mono[13])/8\n",
    "\n",
    "ruDiff_mono = ruMPPAvg_mono - ruMAPAvg_mono\n",
    "\n",
    "roMeans_mono = np.load(\"commonResults/roMeans_mono.npy\")\n",
    "\n",
    "roMPPAvg_mono = (roMeans_mono[0] + roMeans_mono[1] + roMeans_mono[2] + roMeans_mono[3] + roMeans_mono[4] + roMeans_mono[5])/6\n",
    "roMAPAvg_mono = (roMeans_mono[6] + roMeans_mono[7] + roMeans_mono[8] + roMeans_mono[9] + roMeans_mono[10] + roMeans_mono[11] + roMeans_mono[12] + roMeans_mono[13])/8\n",
    "\n",
    "roDiff_mono = roMPPAvg_mono - roMAPAvg_mono\n",
    "\n",
    "siMeans_mono = np.load(\"commonResults/siMeans_mono.npy\")\n",
    "\n",
    "siMPPAvg_mono = (siMeans_mono[0] + siMeans_mono[1] + siMeans_mono[2] + siMeans_mono[3] + siMeans_mono[4] + siMeans_mono[5])/6\n",
    "siMAPAvg_mono = (siMeans_mono[6] + siMeans_mono[7] + siMeans_mono[8] + siMeans_mono[9] + siMeans_mono[10] + siMeans_mono[11] + siMeans_mono[12] + siMeans_mono[13])/8\n",
    "\n",
    "siDiff_mono = siMPPAvg_mono - siMAPAvg_mono\n",
    "\n",
    "neMeans_mono = np.load(\"commonResults/neMeans_mono.npy\")\n",
    "\n",
    "neMPPAvg_mono = (neMeans_mono[0] + neMeans_mono[1] + neMeans_mono[2] + neMeans_mono[3] + neMeans_mono[4] + neMeans_mono[5])/6\n",
    "neMAPAvg_mono = (neMeans_mono[6] + neMeans_mono[7] + neMeans_mono[8] + neMeans_mono[9] + neMeans_mono[10] + neMeans_mono[11] + neMeans_mono[12] + neMeans_mono[13])/8\n",
    "\n",
    "neDiff_mono = neMPPAvg_mono - neMAPAvg_mono"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64b778fe-052e-449f-9b5b-0b7d99c08dd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "etMeans_siamese = np.load(\"commonResults/etMeans_siamese.npy\")\n",
    "\n",
    "etMPPAvg_siamese = (etMeans_siamese[0] + etMeans_siamese[1] + etMeans_siamese[2] + etMeans_siamese[3] + etMeans_siamese[4] + etMeans_siamese[5])/6\n",
    "etMAPAvg_siamese = (etMeans_siamese[6] + etMeans_siamese[7] + etMeans_siamese[8] + etMeans_siamese[9] + etMeans_siamese[10] + etMeans_siamese[11] + etMeans_siamese[12] + etMeans_siamese[13])/8\n",
    "\n",
    "etDiff_siamese = etMPPAvg_siamese - etMAPAvg_siamese\n",
    "\n",
    "ruMeans_siamese = np.load(\"commonResults/ruMeans_siamese.npy\")\n",
    "\n",
    "ruMPPAvg_siamese = (ruMeans_siamese[0] + ruMeans_siamese[1] + ruMeans_siamese[2] + ruMeans_siamese[3] + ruMeans_siamese[4] + ruMeans_siamese[5])/6\n",
    "ruMAPAvg_siamese = (ruMeans_siamese[6] + ruMeans_siamese[7] + ruMeans_siamese[8] + ruMeans_siamese[9] + ruMeans_siamese[10] + ruMeans_siamese[11] + ruMeans_siamese[12] + ruMeans_siamese[13])/8\n",
    "\n",
    "ruDiff_siamese = ruMPPAvg_siamese - ruMAPAvg_siamese\n",
    "\n",
    "roMeans_siamese = np.load(\"commonResults/roMeans_siamese.npy\")\n",
    "\n",
    "roMPPAvg_siamese = (roMeans_siamese[0] + roMeans_siamese[1] + roMeans_siamese[2] + roMeans_siamese[3] + roMeans_siamese[4] + roMeans_siamese[5])/6\n",
    "roMAPAvg_siamese = (roMeans_siamese[6] + roMeans_siamese[7] + roMeans_siamese[8] + roMeans_siamese[9] + roMeans_siamese[10] + roMeans_siamese[11] + roMeans_siamese[12] + roMeans_siamese[13])/8\n",
    "\n",
    "roDiff_siamese = roMPPAvg_siamese - roMAPAvg_siamese\n",
    "\n",
    "siMeans_siamese = np.load(\"commonResults/siMeans_siamese.npy\")\n",
    "\n",
    "siMPPAvg_siamese = (siMeans_siamese[0] + siMeans_siamese[1] + siMeans_siamese[2] + siMeans_siamese[3] + siMeans_siamese[4] + siMeans_siamese[5])/6\n",
    "siMAPAvg_siamese = (siMeans_siamese[6] + siMeans_siamese[7] + siMeans_siamese[8] + siMeans_siamese[9] + siMeans_siamese[10] + siMeans_siamese[11] + siMeans_siamese[12] + siMeans_siamese[13])/8\n",
    "\n",
    "siDiff_siamese = siMPPAvg_siamese - siMAPAvg_siamese\n",
    "\n",
    "neMeans_siamese = np.load(\"commonResults/neMeans_siamese.npy\")\n",
    "\n",
    "neMPPAvg_siamese = (neMeans_siamese[0] + neMeans_siamese[1] + neMeans_siamese[2] + neMeans_siamese[3] + neMeans_siamese[4] + neMeans_siamese[5])/6\n",
    "neMAPAvg_siamese = (neMeans_siamese[6] + neMeans_siamese[7] + neMeans_siamese[8] + neMeans_siamese[9] + neMeans_siamese[10] + neMeans_siamese[11] + neMeans_siamese[12] + neMeans_siamese[13])/8\n",
    "\n",
    "neDiff_siamese = neMPPAvg_siamese - neMAPAvg_siamese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78c65f4a-8051-4aef-a30a-2dc1f48eccfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "etMeans_multi = np.load(\"commonResults/etMeans_multi.npy\")\n",
    "\n",
    "etMPPAvg_multi = (etMeans_multi[0] + etMeans_multi[1] + etMeans_multi[2] + etMeans_multi[3] + etMeans_multi[4] + etMeans_multi[5])/6\n",
    "etMAPAvg_multi = (etMeans_multi[6] + etMeans_multi[7] + etMeans_multi[8] + etMeans_multi[9] + etMeans_multi[10] + etMeans_multi[11] + etMeans_multi[12] + etMeans_multi[13])/8\n",
    "\n",
    "etDiff_multi = etMPPAvg_multi - etMAPAvg_multi\n",
    "\n",
    "ruMeans_multi = np.load(\"commonResults/ruMeans_multi.npy\")\n",
    "\n",
    "ruMPPAvg_multi = (ruMeans_multi[0] + ruMeans_multi[1] + ruMeans_multi[2] + ruMeans_multi[3] + ruMeans_multi[4] + ruMeans_multi[5])/6\n",
    "ruMAPAvg_multi = (ruMeans_multi[6] + ruMeans_multi[7] + ruMeans_multi[8] + ruMeans_multi[9] + ruMeans_multi[10] + ruMeans_multi[11] + ruMeans_multi[12] + ruMeans_multi[13])/8\n",
    "\n",
    "ruDiff_multi = ruMPPAvg_multi - ruMAPAvg_multi\n",
    "\n",
    "roMeans_multi = np.load(\"commonResults/roMeans_multi.npy\")\n",
    "\n",
    "roMPPAvg_multi = (roMeans_multi[0] + roMeans_multi[1] + roMeans_multi[2] + roMeans_multi[3] + roMeans_multi[4] + roMeans_multi[5])/6\n",
    "roMAPAvg_multi = (roMeans_multi[6] + roMeans_multi[7] + roMeans_multi[8] + roMeans_multi[9] + roMeans_multi[10] + roMeans_multi[11] + roMeans_multi[12] + roMeans_multi[13])/8\n",
    "\n",
    "roDiff_multi = roMPPAvg_multi - roMAPAvg_multi\n",
    "\n",
    "siMeans_multi = np.load(\"commonResults/siMeans_multi.npy\")\n",
    "\n",
    "siMPPAvg_multi = (siMeans_multi[0] + siMeans_multi[1] + siMeans_multi[2] + siMeans_multi[3] + siMeans_multi[4] + siMeans_multi[5])/6\n",
    "siMAPAvg_multi = (siMeans_multi[6] + siMeans_multi[7] + siMeans_multi[8] + siMeans_multi[9] + siMeans_multi[10] + siMeans_multi[11] + siMeans_multi[12] + siMeans_multi[13])/8\n",
    "\n",
    "siDiff_multi = siMPPAvg_multi - siMAPAvg_multi\n",
    "\n",
    "neMeans_multi = np.load(\"commonResults/neMeans_multi.npy\")\n",
    "\n",
    "neMPPAvg_multi = (neMeans_multi[0] + neMeans_multi[1] + neMeans_multi[2] + neMeans_multi[3] + neMeans_multi[4] + neMeans_multi[5])/6\n",
    "neMAPAvg_multi = (neMeans_multi[6] + neMeans_multi[7] + neMeans_multi[8] + neMeans_multi[9] + neMeans_multi[10] + neMeans_multi[11] + neMeans_multi[12] + neMeans_multi[13])/8\n",
    "\n",
    "neDiff_multi = neMPPAvg_multi - neMAPAvg_multi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2a7480e-4e9b-41a4-ba51-e0ebfa552019",
   "metadata": {},
   "outputs": [],
   "source": [
    "etMeans_unsupervised = np.load(\"commonResults/etMeans_unsupervised.npy\")\n",
    "\n",
    "etMPPAvg_unsupervised = (etMeans_unsupervised[0] + etMeans_unsupervised[1] + etMeans_unsupervised[2] + etMeans_unsupervised[3] + etMeans_unsupervised[4] + etMeans_unsupervised[5])/6\n",
    "etMAPAvg_unsupervised = (etMeans_unsupervised[6] + etMeans_unsupervised[7] + etMeans_unsupervised[8] + etMeans_unsupervised[9] + etMeans_unsupervised[10] + etMeans_unsupervised[11] + etMeans_unsupervised[12] + etMeans_unsupervised[13])/8\n",
    "\n",
    "etDiff_unsupervised = etMPPAvg_unsupervised - etMAPAvg_unsupervised\n",
    "\n",
    "ruMeans_unsupervised = np.load(\"commonResults/ruMeans_unsupervised.npy\")\n",
    "\n",
    "ruMPPAvg_unsupervised = (ruMeans_unsupervised[0] + ruMeans_unsupervised[1] + ruMeans_unsupervised[2] + ruMeans_unsupervised[3] + ruMeans_unsupervised[4] + ruMeans_unsupervised[5])/6\n",
    "ruMAPAvg_unsupervised = (ruMeans_unsupervised[6] + ruMeans_unsupervised[7] + ruMeans_unsupervised[8] + ruMeans_unsupervised[9] + ruMeans_unsupervised[10] + ruMeans_unsupervised[11] + ruMeans_unsupervised[12] + ruMeans_unsupervised[13])/8\n",
    "\n",
    "ruDiff_unsupervised = ruMPPAvg_unsupervised - ruMAPAvg_unsupervised\n",
    "\n",
    "roMeans_unsupervised = np.load(\"commonResults/roMeans_unsupervised.npy\")\n",
    "\n",
    "roMPPAvg_unsupervised = (roMeans_unsupervised[0] + roMeans_unsupervised[1] + roMeans_unsupervised[2] + roMeans_unsupervised[3] + roMeans_unsupervised[4] + roMeans_unsupervised[5])/6\n",
    "roMAPAvg_unsupervised = (roMeans_unsupervised[6] + roMeans_unsupervised[7] + roMeans_unsupervised[8] + roMeans_unsupervised[9] + roMeans_unsupervised[10] + roMeans_unsupervised[11] + roMeans_unsupervised[12] + roMeans_unsupervised[13])/8\n",
    "\n",
    "roDiff_unsupervised = roMPPAvg_unsupervised - roMAPAvg_unsupervised\n",
    "\n",
    "siMeans_unsupervised = np.load(\"commonResults/siMeans_unsupervised.npy\")\n",
    "\n",
    "siMPPAvg_unsupervised = (siMeans_unsupervised[0] + siMeans_unsupervised[1] + siMeans_unsupervised[2] + siMeans_unsupervised[3] + siMeans_unsupervised[4] + siMeans_unsupervised[5])/6\n",
    "siMAPAvg_unsupervised = (siMeans_unsupervised[6] + siMeans_unsupervised[7] + siMeans_unsupervised[8] + siMeans_unsupervised[9] + siMeans_unsupervised[10] + siMeans_unsupervised[11] + siMeans_unsupervised[12] + siMeans_unsupervised[13])/8\n",
    "\n",
    "siDiff_unsupervised = siMPPAvg_unsupervised - siMAPAvg_unsupervised\n",
    "\n",
    "neMeans_unsupervised = np.load(\"commonResults/neMeans_unsupervised.npy\")\n",
    "\n",
    "neMPPAvg_unsupervised = (neMeans_unsupervised[0] + neMeans_unsupervised[1] + neMeans_unsupervised[2] + neMeans_unsupervised[3] + neMeans_unsupervised[4] + neMeans_unsupervised[5])/6\n",
    "neMAPAvg_unsupervised = (neMeans_unsupervised[6] + neMeans_unsupervised[7] + neMeans_unsupervised[8] + neMeans_unsupervised[9] + neMeans_unsupervised[10] + neMeans_unsupervised[11] + neMeans_unsupervised[12] + neMeans_unsupervised[13])/8\n",
    "\n",
    "neDiff_unsupervised = neMPPAvg_unsupervised - neMAPAvg_unsupervised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b309f5b-7d16-40bb-ab84-389d5278b206",
   "metadata": {},
   "outputs": [],
   "source": [
    "etRanking = [etDiff_mono, etDiff_siamese, etDiff_multi, etDiff_unsupervised]\n",
    "ruRanking = [ruDiff_mono, ruDiff_siamese, ruDiff_multi, ruDiff_unsupervised]\n",
    "roRanking = [roDiff_mono, roDiff_siamese, roDiff_multi, roDiff_unsupervised]\n",
    "siRanking = [siDiff_mono, siDiff_siamese, siDiff_multi, siDiff_unsupervised]\n",
    "neRanking = [neDiff_mono, neDiff_siamese, neDiff_multi, neDiff_unsupervised]\n",
    "\n",
    "rows = [\"MonoTransQuest\", \"SiameseTransQuest\", \"MultiTransQuest\", \"Unsupervised\"] ## Since only 4 implementations are present with the code released.\n",
    "\n",
    "plt.figure(figsize=(9, 5))\n",
    "plt.bar(rows, etRanking)\n",
    "plt.suptitle('Et-En Ranked Systems')\n",
    "plt.show()\n",
    "plt.figure(figsize=(9, 5))\n",
    "plt.bar(rows, ruRanking)\n",
    "plt.suptitle('Ru-En Ranked Systems')\n",
    "plt.show()\n",
    "plt.figure(figsize=(9, 5))\n",
    "plt.bar(rows, siRanking)\n",
    "plt.suptitle('Si-En Ranked Systems')\n",
    "plt.show()\n",
    "plt.figure(figsize=(9, 5))\n",
    "plt.bar(rows, roRanking)\n",
    "plt.suptitle('Ro-En Ranked Systems')\n",
    "plt.show()\n",
    "plt.figure(figsize=(9, 5))\n",
    "plt.bar(rows, neRanking)\n",
    "plt.suptitle('Ne-En Ranked Systems')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}