demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import argparse\n",
    "import json\n",
    "import sys\n",
    "from pathlib import Path\n",
    "import _pickle as cPickle\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "from natsort import natsorted\n",
    "from torch import optim\n",
    "from PIL import Image\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import plotly.graph_objs as go\n",
    "import plotly.offline as py\n",
    "import scipy.io as sio\n",
    "from matplotlib.patches import Rectangle\n",
    "import cv2\n",
    "import matplotlib.image as img\n",
    "import matplotlib.pyplot as plt\n",
    "sys.path.append(\"modules/helpers\")\n",
    "sys.path.append(\"configs\")\n",
    "sys.path.append(\"common\")\n",
    "\n",
    "import robust_loss_pytorch.general\n",
    "from mv_nrsfm_configs import get_dictionary, get_seed\n",
    "from mvnrsfm.neural_prior_modules import (\n",
    "    BlockSparseCodingLayer,\n",
    "    CameraEstimator,\n",
    "    OnPEstimator,\n",
    "    SparseCodeExtractionLayer,\n",
    "    VecSparseCodingLayer,\n",
    ")\n",
    "from util_cameras import normalize_3d_structure, procrustes_align\n",
    "from util_common import get_device, make_dir, parse_boolean, extract_bone_connections, get_colors\n",
    "from util_errors import computeMPJPE, error_ratio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Parameters \"\"\"\n",
    "dataset = \"Monkey\"\n",
    "\n",
    "params = {\"model\": \"models/\" + dataset, \\\n",
    "          \"batch\": 1000, \\\n",
    "          \"dataset\": dataset, \\\n",
    "          \"dataset_metric_multiplier\": 1000, \\\n",
    "          \"img_type\": \".jpg\"\n",
    "          }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MVNRSfM(nn.Module):\n",
    "    \"\"\"\n",
    "    Class containing all the parameters for the MultiView Non-Rigid Structure-from-Motion.\n",
    "    Lifts 2D to 3D\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        params,\n",
    "        GT_Flag=False,\n",
    "        DEVICE=\"cuda\",\n",
    "        num_joints=21,\n",
    "        total_views=4,\n",
    "        total_frames=1000,\n",
    "        dict_list=list(np.arange(125, 7, -10)),\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Initialize the BaseAPI object\n",
    "        :param num_joints: number of joints\n",
    "        :param dict_list: sparse dictionary dimensions\n",
    "\n",
    "        \"\"\"\n",
    "        # Necessary for proper multiple sub-classing\n",
    "        super(MVNRSfM, self).__init__()\n",
    "        self.model_path = params[\"model\"]  # model path or trained weights path\n",
    "        self.from_scratch = False\n",
    "        self.DEVICE = DEVICE\n",
    "        self.num_joints = num_joints\n",
    "        self.total_views = total_views\n",
    "        self.total_frames = total_frames\n",
    "        share_weight = True  # Share weights b/w Encoder and Decoder Dictionaries\n",
    "\n",
    "        # Neural Prior Module\n",
    "        self.sparse_coding_layers = nn.ModuleList(\n",
    "            [VecSparseCodingLayer(dict_list[0], num_joints, share_weight)]\n",
    "        )\n",
    "        for idx in range(1, len(dict_list)):\n",
    "            self.sparse_coding_layers.append(\n",
    "                BlockSparseCodingLayer(dict_list[idx], dict_list[idx - 1], share_weight)\n",
    "            )\n",
    "        self.code_estimator = SparseCodeExtractionLayer(code_dim=dict_list[-1], is_2d=True)\n",
    "        self.GT_Flag = GT_Flag\n",
    "\n",
    "        # Camera Module\n",
    "        self.onp_estimator = OnPEstimator()\n",
    "\n",
    "        # MV-NRSfM Training\n",
    "        if ~self.GT_Flag:\n",
    "            self.unit_scale = True\n",
    "        self.dataset_metric_multiplier = params[\"dataset_metric_multiplier\"]\n",
    "        self.dict_dims = dict_list\n",
    "        self.log_dir = params[\"model\"]\n",
    "        self.print_freq = 50\n",
    "        self.save_freq = 1000\n",
    "        self.lr = 0.0001\n",
    "        self.max_itr = 1000000\n",
    "        self.lr_decay_step = 5000\n",
    "        self.lr_decay_rate = 0.95\n",
    "        self.validation_freq = 1000\n",
    "        self.batch = params[\"model\"]\n",
    "        self.bottleneck_size = dict_list[-1]\n",
    "\n",
    "        # Load model\n",
    "        if self.DEVICE.type == \"cpu\":\n",
    "            self.model = self.load_state_dict(\n",
    "                torch.load(params[\"model\"] + \"/model_best.pth\", map_location=torch.device(\"cpu\"))\n",
    "            )\n",
    "        else:\n",
    "            self.model = self.load_state_dict(torch.load(params[\"model\"] + \"/model_best.pth\"))\n",
    "\n",
    "    def save_model(self, checkpoint_file):\n",
    "        torch.save(self.state_dict(), checkpoint_file)\n",
    "\n",
    "    def get_parameters(self):\n",
    "        return self.model.parameters()\n",
    "\n",
    "    def get_trace3d(self, joint_connections, points3d, point_color=None, line_color=None, name=\"PointCloud\"):\n",
    "        \"\"\"Yields plotly traces for visualization\"\"\"\n",
    "        if point_color is None:\n",
    "            point_color = \"rgb(30, 20, 160)\"\n",
    "        if line_color is None:\n",
    "            line_color = \"rgb(30, 20, 160)\"\n",
    "\n",
    "        # Trace of points\n",
    "        trace_pts = go.Scatter3d(\n",
    "            x=points3d[:, 0],\n",
    "            y=points3d[:, 2],\n",
    "            z=points3d[:, 1],\n",
    "            mode=\"markers\",\n",
    "            name=name,\n",
    "            marker=dict(symbol=\"circle\", size=6, color=point_color),\n",
    "        )\n",
    "\n",
    "        # Trace of line\n",
    "        xlines, ylines, zlines = [], [], []\n",
    "        for line in joint_connections:\n",
    "            for point in line:\n",
    "                xlines.append(points3d[point, 0])\n",
    "                ylines.append(points3d[point, 2])\n",
    "                zlines.append(points3d[point, 1])\n",
    "            xlines.append(None)\n",
    "            ylines.append(None)\n",
    "            zlines.append(None)\n",
    "        trace_lines = go.Scatter3d(\n",
    "            x=xlines, y=ylines, z=zlines, mode=\"lines\", name=name, line=dict(width=6, color=line_color)\n",
    "        )\n",
    "\n",
    "        return [trace_pts, trace_lines]\n",
    "\n",
    "    def get_trace2d(self, points2d, point_color=None, line_color=None, name=\"PointCloud\"):\n",
    "        \"\"\" Yields plotly traces for visualization \"\"\"\n",
    "        if point_color is None:\n",
    "            point_color = \"rgb(30, 20, 160)\"\n",
    "        if line_color is None:\n",
    "            line_color = \"rgb(30, 20, 160)\"\n",
    "\n",
    "        # Trace of points\n",
    "        trace_pts = go.Scatter(x=points2d[:,0], y=points2d[:,1],\n",
    "                                mode=\"markers\", name=name,\n",
    "                                marker=dict(symbol=\"circle\", size=6, color=point_color))\n",
    "\n",
    "        # Trace of line\n",
    "        xlines, ylines = [],[]\n",
    "        for line in joint_connections:\n",
    "            for point in line:\n",
    "                xlines.append(points2d[point, 0])\n",
    "                ylines.append(points2d[point, 1])\t\t\t\n",
    "            xlines.append(None)\n",
    "            ylines.append(None)\t\t\n",
    "        trace_lines = go.Scatter(x=xlines, y=ylines,\n",
    "                                mode=\"lines\", name=name,\n",
    "                                line=dict(width=6,color=line_color))\n",
    "\n",
    "        return [trace_pts, trace_lines]\n",
    "\n",
    "    def get_figure2d(self, points2d, gt=None, range_scale=1):\n",
    "        \"\"\" Yields plotly figure for visualization \"\"\"\n",
    "        color_blue = \"rgb(0, 0, 255)\"\n",
    "        color_red = \"rgb(255, 0, 0)\"        \n",
    "        traces = self.get_trace2d(points2d, color_blue, color_blue, \"Predicted KP\")\n",
    "        if gt is not None:\n",
    "            traces += self.get_trace2d(gt, color_red, color_red, \"Groundtruth KP\")\n",
    "\n",
    "        layout = go.Layout(scene=dict(\n",
    "                                    aspectratio=dict(x=0.8, y=0.8),\t\t\t\t\t\t\t\t  \n",
    "                                    xaxis=dict(range=(-0.4 * range_scale, 0.4 * range_scale)),\n",
    "                                    yaxis=dict(range=(-0.4 * range_scale, 0.4 * range_scale))),\n",
    "                        width=700,\n",
    "                        margin=dict(r=20, l=10, b=10, t=10))\n",
    "        return go.Figure(data=traces, layout=layout)\t\n",
    "\n",
    "\n",
    "\n",
    "    def get_figure3d(\n",
    "        self, joint_connections, input_cam_x, input_cam_y, points3d, points3d_gt, range_scale=2500\n",
    "    ):\n",
    "        \"\"\"Yields plotly figure for visualization\"\"\"\n",
    "        color_palette = get_colors()\n",
    "        color_blue, color_red = color_palette[\"color_blue\"], color_palette[\"color_red\"]\n",
    "        traces = self.get_trace3d(joint_connections, points3d, color_blue, color_blue, \"Predicted KP\")\n",
    "        if points3d_gt is not None:\n",
    "            traces += self.get_trace3d(\n",
    "                joint_connections, points3d_gt, color_red, color_red, \"Groundtruth KP\"\n",
    "            )\n",
    "\n",
    "        layout = go.Layout(\n",
    "            scene=dict(\n",
    "                aspectratio=dict(x=0.8, y=0.8, z=2),\n",
    "                xaxis=dict(range=(-0.4 * range_scale, 0.4 * range_scale)),\n",
    "                yaxis=dict(range=(-0.4 * range_scale, 0.4 * range_scale)),\n",
    "                zaxis=dict(range=(-1 * range_scale, 1 * range_scale)),\n",
    "            ),\n",
    "            width=700,\n",
    "            margin=dict(r=20, l=10, b=10, t=10),\n",
    "            scene_camera=dict(\n",
    "                up=dict(x=0, y=0, z=1),\n",
    "                center=dict(x=0, y=0, z=0),\n",
    "                eye=dict(x=input_cam_x, y=input_cam_y, z=1.25),\n",
    "            ),\n",
    "        )\n",
    "        return go.Figure(data=traces, layout=layout)\n",
    "\n",
    "    def gen_image(self, im, W_GT, W_Predictions, joint_connections):\n",
    "        \n",
    "        subplots = plt.subplots(1, 2)\n",
    "\n",
    "        fig = subplots[0]\n",
    "        axes = subplots[1]\n",
    "\n",
    "        for idx in range(2):        \n",
    "            axes[idx].imshow(im)\n",
    "\n",
    "            if idx == 0:\n",
    "                W = W_GT\n",
    "                color_kpts = \"r\"\n",
    "                fontdict = {\"fontsize\": 50, \"color\": color_kpts}                \n",
    "                axes[idx].axes.set_title(\"Grountruth\", fontdict)\n",
    "            elif idx == 1:\n",
    "                W = W_Predictions\n",
    "                color_kpts = \"b\"\n",
    "                fontdict = {\"fontsize\": 50, \"color\": color_kpts}\n",
    "                axes[idx].axes.set_title(\"Prediction\", fontdict)\n",
    "            axes[idx].scatter(x=W[:, 0], y=W[:, 1], c=color_kpts, s=40)\n",
    "            xlines, ylines = extract_bone_connections(W, joint_connections)\n",
    "            axes[idx].plot(xlines, ylines, c=color_kpts, linewidth=4)            \n",
    "            axes[idx].axes.xaxis.set_visible(False)\n",
    "            axes[idx].axes.yaxis.set_visible(False)\n",
    "\n",
    "        figure = plt.gcf()\n",
    "        figure.set_size_inches(38, 36)\n",
    "\n",
    "        return figure\n",
    "\n",
    "\n",
    "    def forward(self, pts_2d_packet):\n",
    "        layer_num = len(self.sparse_coding_layers)\n",
    "        code_inner = (\n",
    "            torch.from_numpy(\n",
    "                np.zeros((len(pts_2d_packet), pts_2d_packet[0].shape[0], self.dict_dims[-1], 1, 1))\n",
    "            )\n",
    "            .float()\n",
    "            .to(self.DEVICE)\n",
    "        )\n",
    "\n",
    "        \"\"\" Encode the 2D representation. \"\"\"\n",
    "        camera_matrix = []\n",
    "        for outer_idx in range(len(pts_2d_packet)):\n",
    "            flag_nan = ~torch.any(torch.any(torch.isnan(pts_2d_packet[outer_idx]), dim=1), dim=1)\n",
    "            code_block = pts_2d_packet[outer_idx]\n",
    "            code_block = code_block[flag_nan]\n",
    "            indices_non_nan = torch.nonzero(flag_nan.float()).view(-1)\n",
    "            for idx in range(layer_num):\n",
    "                code_block = self.sparse_coding_layers[idx].encode_with_cam(code_block)\n",
    "\n",
    "            code_inner[outer_idx, indices_non_nan, :, :, :] = (\n",
    "                self.code_estimator(code_block).float().to(self.DEVICE)\n",
    "            )\n",
    "\n",
    "        \"\"\" Enforce multi-view consistency by pooling the codes from all the views. \"\"\"\n",
    "        code_pooled = torch.sum(code_inner, dim=0) / len(pts_2d_packet)\n",
    "        code_canon = code_pooled.clone()\n",
    "\n",
    "        \"\"\" Decode the pooled code and generate 3D structure in a canonical frame. \"\"\"\n",
    "        for idx in range(layer_num - 1, 0, -1):\n",
    "            code_pooled = self.sparse_coding_layers[idx].decode(code_pooled)\n",
    "        pts_recon_canonical, dict_final_pass = self.sparse_coding_layers[0].decode(code_pooled)\n",
    "        W_cat = torch.cat((pts_2d_packet), dim=0)\n",
    "        S_cat_pred = torch.cat([pts_recon_canonical] * len(pts_2d_packet))\n",
    "        flag_nan = ~torch.any(torch.any(torch.isnan(W_cat), dim=1), dim=1)\n",
    "\n",
    "        \"\"\" Estimate the camera using least squares algebraic solution of OnP \"\"\"\n",
    "        pts_2d_reproj, pts_3d_recon, cam_matrix, rotation_matrix = self.onp_estimator(\n",
    "            W_cat[flag_nan], S_cat_pred[flag_nan]\n",
    "        )\n",
    "\n",
    "        prediction = (\n",
    "            pts_3d_recon,\n",
    "            pts_2d_reproj,\n",
    "            W_cat,\n",
    "            code_canon,\n",
    "            cam_matrix,\n",
    "            pts_recon_canonical,\n",
    "        )\n",
    "\n",
    "        return prediction, flag_nan\n",
    "\n",
    "    def train(self, adaptive, input):\n",
    "        prediction, flag_nan = self.forward(input)\n",
    "        S_Pred, W_Pred, W_Input = (prediction[0], prediction[1], prediction[2])\n",
    "        reprojection_loss = torch.mean(\n",
    "            adaptive.lossfun(\n",
    "                W_Pred.view(W_Pred.shape[0], W_Pred.shape[1] * W_Pred.shape[2])\n",
    "                - W_Input[flag_nan].view(\n",
    "                    W_Input[flag_nan].shape[0],\n",
    "                    W_Input[flag_nan].shape[1] * W_Input[flag_nan].shape[2],\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "        return reprojection_loss\n",
    "\n",
    "    def validation(self, input):\n",
    "        with torch.no_grad():\n",
    "            W = input[0]\n",
    "            T_Offset = torch.cat(input[1], dim=0)\n",
    "            W_GT = torch.cat(input[2], dim=0)\n",
    "            if self.GT_Flag:\n",
    "                S_GT = torch.cat(input[3], dim=0)\n",
    "            W_, S_, Errors_ = [], [], []\n",
    "            prediction, flag_nan = self.forward(W)\n",
    "\n",
    "            # All of the fields below are non-nan values. Fill them back in their respective size matrices.\n",
    "            S_Pred, W_Pred, W_Input, code, cam, S_Canon = (\n",
    "                prediction[0],\n",
    "                prediction[1],\n",
    "                prediction[2],\n",
    "                prediction[3],\n",
    "                prediction[4],\n",
    "                prediction[5],\n",
    "            )\n",
    "\n",
    "            # Define full size\n",
    "            dim_1, dim_2 = torch.cat(W, dim=0).shape[0], torch.cat(W, dim=0).shape[1]\n",
    "\n",
    "            # Fill the full size matrices\n",
    "            W_Pred = W_Pred + T_Offset[flag_nan]\n",
    "            W_Input[flag_nan] = W_Input[flag_nan] + T_Offset[flag_nan]\n",
    "            S_Pred_, W_Pred_ = (\n",
    "                torch.zeros((dim_1, dim_2, 3)).to(self.DEVICE).float(),\n",
    "                torch.zeros((dim_1, dim_2, 2)).to(self.DEVICE),\n",
    "            )\n",
    "            S_Pred_[:], W_Pred_[:] = torch.nan, torch.nan\n",
    "            S_Pred_[flag_nan], W_Pred_[flag_nan] = S_Pred, W_Pred\n",
    "\n",
    "            # Procrustes-Align if G.T. is available and pack the matrices to deliver back.\n",
    "            if self.GT_Flag:\n",
    "                # 3D Error\n",
    "                S_Pred_ = procrustes_align(S_Pred_, S_GT)\n",
    "                PA_MPJPE_3D = (\n",
    "                    computeMPJPE(S_Pred_[flag_nan], S_GT[flag_nan]) * self.dataset_metric_multiplier\n",
    "                )\n",
    "                S_GT = S_GT.reshape(self.total_views, self.total_frames, self.num_joints, 3)\n",
    "            else:\n",
    "                S_Pred_ = normalize_3d_structure(S_Pred_)\n",
    "                PA_MPJPE_3D = None\n",
    "\n",
    "            non_nans = np.argwhere(\n",
    "                ~np.any(np.any(np.isnan(W_Input.detach().cpu().numpy()), axis=1), axis=1) == True\n",
    "            )[:, 0]\n",
    "\n",
    "            W_GT = W_GT + T_Offset\n",
    "\n",
    "            error_2D = (\n",
    "                error_ratio(W_Pred_[non_nans], W_GT[non_nans]) * self.dataset_metric_multiplier\n",
    "            )\n",
    "\n",
    "            errors = {\"2D_Error_Ratio\": error_2D, \"3D_PA_MPJPE\": PA_MPJPE_3D}\n",
    "            S_Pred_ = S_Pred_.reshape(self.total_views, self.total_frames, self.num_joints, 3)\n",
    "            W_Pred_ = W_Pred_.reshape(self.total_views, self.total_frames, self.num_joints, 2)\n",
    "            W_GT = W_GT.reshape(self.total_views, self.total_frames, self.num_joints, 2)\n",
    "            W_Input = W_Input.reshape(self.total_views, self.total_frames, self.num_joints, 2)\n",
    "\n",
    "        return errors, W_Pred_, S_Pred_, W_GT\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" GPU Device allotment \"\"\"\n",
    "DEVICE = get_device()\n",
    "\n",
    "\"\"\" Extract joint connections \"\"\"\n",
    "with open(\"configs/joints/\" + str(dataset) + \".json\", \"r\") as fid:\n",
    "    joints_data = json.load(fid)\n",
    "joint_connections = joints_data[\"joint_connections\"]\n",
    "range_scale = joints_data[\"range_scale\"]\n",
    "rigid_rotation = joints_data[\"R\"]\n",
    "\n",
    "\"\"\" Extract annotation file paths \"\"\"\n",
    "annot_path = \"data/\" + str(dataset) + \"/annot/\"\n",
    "pkl_names = []\n",
    "for path in Path(annot_path).rglob(\"*.pkl\"):\n",
    "    pkl_names.append(path)\n",
    "pkl_names = [Path(p) for p in natsorted([str(p) for p in pkl_names])]\n",
    "\n",
    "\"\"\" Extract total number of views \"\"\"\n",
    "total_views = len(pkl_names)\n",
    "\n",
    "\"\"\" Extract G.T. 2D and 3D if available, else only 2D \"\"\"\n",
    "\"\"\" Find total number of frames \"\"\"\n",
    "with open(str(pkl_names[0]), \"rb\") as fid:\n",
    "    pickled_data = cPickle.load(fid)\n",
    "total_frames = len(pickled_data)\n",
    "\n",
    "\"\"\" Check whether the data has G.T. 2D and 3D \"\"\"\n",
    "if \"S_GT\" in pickled_data[0]:\n",
    "    GT_Flag = True\n",
    "else:\n",
    "    GT_Flag = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Load the data \"\"\"\n",
    "\"\"\" Find number of joints/keypoints \"\"\"\n",
    "num_joints = pickled_data[0][\"W_GT\"].shape[0]\n",
    "\n",
    "W, W_GT, S_GT, T_Offset, confidence = [], [], [], [], []\n",
    "\n",
    "\"\"\" Iterate over views \"\"\"\n",
    "for cam_idx in range(total_views):\n",
    "    with open(pkl_names[cam_idx], \"rb\") as fid:\n",
    "        pickled_data = cPickle.load(fid)\n",
    "    # Iterate over frames\n",
    "    W_, W_GT_, S_GT_, T_Offset_, confidence_ = [], [], [], [], []\n",
    "    for frame_idx in range(total_frames):\n",
    "        W_.append(pickled_data[frame_idx][\"W_GT\"])\n",
    "        W_GT_.append(pickled_data[frame_idx][\"W_GT\"])\n",
    "        if GT_Flag:\n",
    "            S_GT_.append(pickled_data[frame_idx][\"S_GT\"])\n",
    "    W_ = np.asarray(W_)\n",
    "    W_GT_ = np.asarray(W_GT_)\n",
    "    S_GT_ = np.asarray(S_GT_)\n",
    "\n",
    "    non_nans = np.argwhere(np.asarray(confidence_) == False)[:, 0].tolist()\n",
    "    W_[non_nans, :, :] = np.nan\n",
    "\n",
    "    # Remove translation: Enforce object-centric assumptipn\n",
    "    T_Offset_ = W_GT_.mean(1, keepdims=True)\n",
    "    W_ = W_ - W_.mean(1, keepdims=True)\n",
    "    W_GT_ = W_GT_ - T_Offset_\n",
    "    if GT_Flag:\n",
    "        S_GT_ = S_GT_ - S_GT_.mean(1, keepdims=True)\n",
    "\n",
    "    W.append(torch.from_numpy(W_).to(DEVICE).float())\n",
    "    W_GT.append(torch.from_numpy(W_GT_).to(DEVICE).float())\n",
    "    S_GT.append(torch.from_numpy(S_GT_).to(DEVICE).float())\n",
    "    T_Offset.append(torch.from_numpy(T_Offset_).to(DEVICE).float())\n",
    "\n",
    "data_loaded = (W, T_Offset, W_GT, S_GT)\n",
    "\n",
    "\n",
    "\"\"\" Load rotation, scale, and images if available \"\"\"\n",
    "R = rigid_rotation\n",
    "mat_contents = sio.loadmat(\"common/cam_25_8.mat\")\n",
    "cam_x = mat_contents[\"x_final\"][0].tolist()\n",
    "cam_y = mat_contents[\"y_final\"][0].tolist()\n",
    "image_paths_cam = []\n",
    "for cam_idx in range(total_views):\n",
    "    image_paths_cam_ = []\n",
    "    for path in Path(\"data/\" + dataset + \"/images/CAM_\" + str(cam_idx + 1) + \"/\").rglob(\n",
    "        \"*\" + str(params[\"img_type\"])\n",
    "    ):\n",
    "        image_paths_cam_.append(path)\n",
    "    image_paths_cam_ = [Path(p) for p in natsorted([str(p) for p in image_paths_cam_])]\n",
    "    image_paths_cam.append(image_paths_cam_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" MV-NRSfM Dictionary \"\"\"\n",
    "dict_list = get_dictionary()\n",
    "\n",
    "\"\"\" Load the Adaptive Loss function Kernel \"\"\"\n",
    "adaptive = robust_loss_pytorch.adaptive.AdaptiveLossFunction(\n",
    "    num_dims=num_joints * 2, float_dtype=torch.float32, device=DEVICE\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MVNRSfM(\n",
       "  (sparse_coding_layers): ModuleList(\n",
       "    (0): VecSparseCodingLayer()\n",
       "    (1): BlockSparseCodingLayer()\n",
       "    (2): BlockSparseCodingLayer()\n",
       "    (3): BlockSparseCodingLayer()\n",
       "    (4): BlockSparseCodingLayer()\n",
       "    (5): BlockSparseCodingLayer()\n",
       "    (6): BlockSparseCodingLayer()\n",
       "    (7): BlockSparseCodingLayer()\n",
       "    (8): BlockSparseCodingLayer()\n",
       "    (9): BlockSparseCodingLayer()\n",
       "    (10): BlockSparseCodingLayer()\n",
       "    (11): BlockSparseCodingLayer()\n",
       "    (12): BlockSparseCodingLayer()\n",
       "    (13): BlockSparseCodingLayer()\n",
       "    (14): BlockSparseCodingLayer()\n",
       "    (15): BlockSparseCodingLayer()\n",
       "    (16): BlockSparseCodingLayer()\n",
       "    (17): BlockSparseCodingLayer()\n",
       "    (18): BlockSparseCodingLayer()\n",
       "  )\n",
       "  (code_estimator): SparseCodeExtractionLayer(\n",
       "    (fc_layer): Conv2d(8, 8, kernel_size=(3, 2), stride=(1, 1), bias=False)\n",
       "  )\n",
       "  (onp_estimator): OnPEstimator()\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" Initialize the MV-NRSfM Kernel Class \"\"\"\n",
    "mv_nrsfm_kernel = MVNRSfM(\n",
    "    params,\n",
    "    GT_Flag,\n",
    "    DEVICE,\n",
    "    num_joints=num_joints,\n",
    "    total_views=total_views,\n",
    "    total_frames=total_frames,\n",
    "    dict_list=dict_list,\n",
    ")\n",
    "\n",
    "mv_nrsfm_kernel.to(DEVICE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------\n",
      "3D PA-MPJPE: 69.44mm | 2D Error Ratio: 3.36\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Run inference / validation / demo \"\"\"\n",
    "Errors, Predictions_2D, Predictions_3D, GT_2D = mv_nrsfm_kernel.validation(data_loaded)\n",
    "Predictions_2D = Predictions_2D.detach().cpu().numpy()\n",
    "Predictions_3D = Predictions_3D.detach().cpu().numpy()\n",
    "GT_3D = data_loaded[3][0].detach().cpu().numpy()\n",
    "GT_2D = GT_2D.detach().cpu().numpy()\n",
    "print(\"----------------------------------------------------------------------\")\n",
    "if GT_Flag:\n",
    "    best_error = Errors[\"3D_PA_MPJPE\"]\n",
    "    print(\n",
    "        \"3D PA-MPJPE: {:.2f}mm | 2D Error Ratio: {:.2f}\".format(\n",
    "            best_error, Errors[\"2D_Error_Ratio\"]\n",
    "        )\n",
    "    )\n",
    "    best_error_2D = Errors[\"2D_Error_Ratio\"]\n",
    "else:\n",
    "    best_error = Errors[\"2D_Error_Ratio\"]\n",
    "    print(\"2D Error Ratio: {:.2f}\".format(best_error))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "linkText": "Export to plot.ly",
        "plotlyServerURL": "https://plot.ly",
        "showLink": false
       },
       "data": [
        {
         "marker": {
          "color": "rgb(0, 0, 255)",
          "size": 6,
          "symbol": "circle"
         },
         "mode": "markers",
         "name": "Predicted KP",
         "type": "scatter3d",
         "x": [
          0.6155094504356384,
          0.4624297618865967,
          0.40621140599250793,
          0.28638705611228943,
          -0.0016576386988162994,
          0.35975223779678345,
          0.648514986038208,
          -0.4860907793045044,
          -0.2902878522872925,
          -0.38980549573898315,
          -0.37972843647003174,
          -0.620324969291687,
          -0.6137182712554932
         ],
         "y": [
          0.21507741510868073,
          0.1538248062133789,
          -0.00044670701026916504,
          -0.08312320709228516,
          -0.14200866222381592,
          0.23157192766666412,
          0.08355063199996948,
          -0.11019781231880188,
          -0.1496761292219162,
          -0.2313230037689209,
          0.08377712965011597,
          -0.025824427604675293,
          -0.023563534021377563
         ],
         "z": [
          0.3891376256942749,
          0.5051054358482361,
          0.21418023109436035,
          0.21758529543876648,
          -0.396151602268219,
          0.22501124441623688,
          -0.5101296305656433,
          0.2704514265060425,
          -0.0503377765417099,
          -0.36818069219589233,
          -0.10305508226156235,
          -0.376436710357666,
          -0.012797847390174866
         ]
        },
        {
         "line": {
          "color": "rgb(0, 0, 255)",
          "width": 6
         },
         "mode": "lines",
         "name": "Predicted KP",
         "type": "scatter3d",
         "x": [
          0.6155094504356384,
          0.4624297618865967,
          null,
          0.4624297618865967,
          0.40621140599250793,
          null,
          0.40621140599250793,
          0.28638705611228943,
          null,
          0.28638705611228943,
          -0.0016576386988162994,
          null,
          0.40621140599250793,
          0.35975223779678345,
          null,
          0.35975223779678345,
          0.648514986038208,
          null,
          0.40621140599250793,
          -0.4860907793045044,
          null,
          -0.4860907793045044,
          -0.2902878522872925,
          null,
          -0.2902878522872925,
          -0.38980549573898315,
          null,
          -0.4860907793045044,
          -0.37972843647003174,
          null,
          -0.37972843647003174,
          -0.620324969291687,
          null,
          -0.4860907793045044,
          -0.6137182712554932,
          null
         ],
         "y": [
          0.21507741510868073,
          0.1538248062133789,
          null,
          0.1538248062133789,
          -0.00044670701026916504,
          null,
          -0.00044670701026916504,
          -0.08312320709228516,
          null,
          -0.08312320709228516,
          -0.14200866222381592,
          null,
          -0.00044670701026916504,
          0.23157192766666412,
          null,
          0.23157192766666412,
          0.08355063199996948,
          null,
          -0.00044670701026916504,
          -0.11019781231880188,
          null,
          -0.11019781231880188,
          -0.1496761292219162,
          null,
          -0.1496761292219162,
          -0.2313230037689209,
          null,
          -0.11019781231880188,
          0.08377712965011597,
          null,
          0.08377712965011597,
          -0.025824427604675293,
          null,
          -0.11019781231880188,
          -0.023563534021377563,
          null
         ],
         "z": [
          0.3891376256942749,
          0.5051054358482361,
          null,
          0.5051054358482361,
          0.21418023109436035,
          null,
          0.21418023109436035,
          0.21758529543876648,
          null,
          0.21758529543876648,
          -0.396151602268219,
          null,
          0.21418023109436035,
          0.22501124441623688,
          null,
          0.22501124441623688,
          -0.5101296305656433,
          null,
          0.21418023109436035,
          0.2704514265060425,
          null,
          0.2704514265060425,
          -0.0503377765417099,
          null,
          -0.0503377765417099,
          -0.36818069219589233,
          null,
          0.2704514265060425,
          -0.10305508226156235,
          null,
          -0.10305508226156235,
          -0.376436710357666,
          null,
          0.2704514265060425,
          -0.012797847390174866,
          null
         ]
        },
        {
         "marker": {
          "color": "rgb(255, 0, 0)",
          "size": 6,
          "symbol": "circle"
         },
         "mode": "markers",
         "name": "Groundtruth KP",
         "type": "scatter3d",
         "x": [
          0.5792799592018127,
          0.4474405348300934,
          0.39271605014801025,
          0.28591200709342957,
          -0.02223011665046215,
          0.34216850996017456,
          0.5998197793960571,
          -0.4391074478626251,
          -0.2726421654224396,
          -0.3945310413837433,
          -0.34479013085365295,
          -0.6082677841186523,
          -0.5657681226730347
         ],
         "y": [
          0.24408286809921265,
          0.16929858922958374,
          -0.01573973521590233,
          -0.12065654247999191,
          -0.20183521509170532,
          0.24558943510055542,
          0.0868491604924202,
          -0.09630756825208664,
          -0.1710382103919983,
          -0.277926504611969,
          0.12741726636886597,
          -0.004262263886630535,
          0.014528696425259113
         ],
         "z": [
          0.42070645093917847,
          0.5249091982841492,
          0.21340839564800262,
          0.21245042979717255,
          -0.46126681566238403,
          0.2574874758720398,
          -0.5354235768318176,
          0.2876209616661072,
          -0.06197817623615265,
          -0.4321461319923401,
          -0.06755168735980988,
          -0.3753975033760071,
          0.01718108542263508
         ]
        },
        {
         "line": {
          "color": "rgb(255, 0, 0)",
          "width": 6
         },
         "mode": "lines",
         "name": "Groundtruth KP",
         "type": "scatter3d",
         "x": [
          0.5792799592018127,
          0.4474405348300934,
          null,
          0.4474405348300934,
          0.39271605014801025,
          null,
          0.39271605014801025,
          0.28591200709342957,
          null,
          0.28591200709342957,
          -0.02223011665046215,
          null,
          0.39271605014801025,
          0.34216850996017456,
          null,
          0.34216850996017456,
          0.5998197793960571,
          null,
          0.39271605014801025,
          -0.4391074478626251,
          null,
          -0.4391074478626251,
          -0.2726421654224396,
          null,
          -0.2726421654224396,
          -0.3945310413837433,
          null,
          -0.4391074478626251,
          -0.34479013085365295,
          null,
          -0.34479013085365295,
          -0.6082677841186523,
          null,
          -0.4391074478626251,
          -0.5657681226730347,
          null
         ],
         "y": [
          0.24408286809921265,
          0.16929858922958374,
          null,
          0.16929858922958374,
          -0.01573973521590233,
          null,
          -0.01573973521590233,
          -0.12065654247999191,
          null,
          -0.12065654247999191,
          -0.20183521509170532,
          null,
          -0.01573973521590233,
          0.24558943510055542,
          null,
          0.24558943510055542,
          0.0868491604924202,
          null,
          -0.01573973521590233,
          -0.09630756825208664,
          null,
          -0.09630756825208664,
          -0.1710382103919983,
          null,
          -0.1710382103919983,
          -0.277926504611969,
          null,
          -0.09630756825208664,
          0.12741726636886597,
          null,
          0.12741726636886597,
          -0.004262263886630535,
          null,
          -0.09630756825208664,
          0.014528696425259113,
          null
         ],
         "z": [
          0.42070645093917847,
          0.5249091982841492,
          null,
          0.5249091982841492,
          0.21340839564800262,
          null,
          0.21340839564800262,
          0.21245042979717255,
          null,
          0.21245042979717255,
          -0.46126681566238403,
          null,
          0.21340839564800262,
          0.2574874758720398,
          null,
          0.2574874758720398,
          -0.5354235768318176,
          null,
          0.21340839564800262,
          0.2876209616661072,
          null,
          0.2876209616661072,
          -0.06197817623615265,
          null,
          -0.06197817623615265,
          -0.4321461319923401,
          null,
          0.2876209616661072,
          -0.06755168735980988,
          null,
          -0.06755168735980988,
          -0.3753975033760071,
          null,
          0.2876209616661072,
          0.01718108542263508,
          null
         ]
        }
       ],
       "layout": {
        "margin": {
         "b": 10,
         "l": 10,
         "r": 20,
         "t": 10
        },
        "scene": {
         "aspectratio": {
          "x": 0.8,
          "y": 0.8,
          "z": 2
         },
         "camera": {
          "center": {
           "x": 0,
           "y": 0,
           "z": 0
          },
          "eye": {
           "x": -1.7678,
           "y": 0,
           "z": 1.25
          },
          "up": {
           "x": 0,
           "y": 0,
           "z": 1
          }
         },
         "xaxis": {
          "range": [
           -1,
           1
          ]
         },
         "yaxis": {
          "range": [
           -1,
           1
          ]
         },
         "zaxis": {
          "range": [
           -2.5,
           2.5
          ]
         }
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 700
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\" Plot the 3D predictions \"\"\"\n",
    "frame_idx = 86\n",
    "fig = mv_nrsfm_kernel.get_figure3d(\n",
    "    joint_connections,\n",
    "    cam_x[0],\n",
    "    cam_y[0],\n",
    "    Predictions_3D[0, frame_idx, :, :] @ R,\n",
    "    GT_3D[frame_idx, :, :] @ R,\n",
    "    range_scale=range_scale,\n",
    ")\n",
    "py.iplot(fig)\n",
    "\n",
    "## P.S. If the 3D structure is out of frame, please reduce the `range_scale` parameter.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "linkText": "Export to plot.ly",
        "plotlyServerURL": "https://plot.ly",
        "showLink": false
       },
       "data": [
        {
         "marker": {
          "color": "rgb(0, 0, 255)",
          "size": 6,
          "symbol": "circle"
         },
         "mode": "markers",
         "name": "Predicted KP",
         "type": "scatter",
         "x": [
          64.36308288574219,
          81.14944458007812,
          90.03776550292969,
          103.8968276977539,
          131.36758422851562,
          87.95893859863281,
          62.19738006591797,
          179.00381469726562,
          160.3484344482422,
          171.25579833984375,
          162.16403198242188,
          187.5455322265625,
          187.94789123535156
         ],
         "y": [
          100.64729309082031,
          89.83698272705078,
          111.58274841308594,
          109.84977722167969,
          168.03366088867188,
          119.89641571044922,
          177.88194274902344,
          114.72652435302734,
          139.9961700439453,
          167.52818298339844,
          154.79856872558594,
          179.1649169921875,
          145.75457763671875
         ]
        },
        {
         "line": {
          "color": "rgb(0, 0, 255)",
          "width": 6
         },
         "mode": "lines",
         "name": "Predicted KP",
         "type": "scatter",
         "x": [
          64.36308288574219,
          81.14944458007812,
          null,
          81.14944458007812,
          90.03776550292969,
          null,
          90.03776550292969,
          103.8968276977539,
          null,
          103.8968276977539,
          131.36758422851562,
          null,
          90.03776550292969,
          87.95893859863281,
          null,
          87.95893859863281,
          62.19738006591797,
          null,
          90.03776550292969,
          179.00381469726562,
          null,
          179.00381469726562,
          160.3484344482422,
          null,
          160.3484344482422,
          171.25579833984375,
          null,
          179.00381469726562,
          162.16403198242188,
          null,
          162.16403198242188,
          187.5455322265625,
          null,
          179.00381469726562,
          187.94789123535156,
          null
         ],
         "y": [
          100.64729309082031,
          89.83698272705078,
          null,
          89.83698272705078,
          111.58274841308594,
          null,
          111.58274841308594,
          109.84977722167969,
          null,
          109.84977722167969,
          168.03366088867188,
          null,
          111.58274841308594,
          119.89641571044922,
          null,
          119.89641571044922,
          177.88194274902344,
          null,
          111.58274841308594,
          114.72652435302734,
          null,
          114.72652435302734,
          139.9961700439453,
          null,
          139.9961700439453,
          167.52818298339844,
          null,
          114.72652435302734,
          154.79856872558594,
          null,
          154.79856872558594,
          179.1649169921875,
          null,
          114.72652435302734,
          145.75457763671875,
          null
         ]
        },
        {
         "marker": {
          "color": "rgb(255, 0, 0)",
          "size": 6,
          "symbol": "circle"
         },
         "mode": "markers",
         "name": "Groundtruth KP",
         "type": "scatter",
         "x": [
          64,
          81,
          90,
          104,
          132,
          88,
          62,
          179,
          160,
          171,
          162,
          188,
          188
         ],
         "y": [
          101,
          90,
          112,
          110,
          168,
          120,
          177,
          114,
          140,
          168,
          155,
          179,
          146
         ]
        },
        {
         "line": {
          "color": "rgb(255, 0, 0)",
          "width": 6
         },
         "mode": "lines",
         "name": "Groundtruth KP",
         "type": "scatter",
         "x": [
          64,
          81,
          null,
          81,
          90,
          null,
          90,
          104,
          null,
          104,
          132,
          null,
          90,
          88,
          null,
          88,
          62,
          null,
          90,
          179,
          null,
          179,
          160,
          null,
          160,
          171,
          null,
          179,
          162,
          null,
          162,
          188,
          null,
          179,
          188,
          null
         ],
         "y": [
          101,
          90,
          null,
          90,
          112,
          null,
          112,
          110,
          null,
          110,
          168,
          null,
          112,
          120,
          null,
          120,
          177,
          null,
          112,
          114,
          null,
          114,
          140,
          null,
          140,
          168,
          null,
          114,
          155,
          null,
          155,
          179,
          null,
          114,
          146,
          null
         ]
        }
       ],
       "layout": {
        "margin": {
         "b": 10,
         "l": 10,
         "r": 20,
         "t": 10
        },
        "scene": {
         "aspectratio": {
          "x": 0.8,
          "y": 0.8
         },
         "xaxis": {
          "range": [
           -0.4,
           0.4
          ]
         },
         "yaxis": {
          "range": [
           -0.4,
           0.4
          ]
         }
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 700
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\" Plot the 2D predictions \"\"\"\n",
    "\"\"\" P.S.: Please choose the camera view \"\"\"\n",
    "camera_view = 0\n",
    "if len(image_paths_cam[0]) > 0:\n",
    "    im = img.imread(image_paths_cam[camera_view][frame_idx])\n",
    "    fig = mv_nrsfm_kernel.gen_image(\n",
    "        im,\n",
    "        GT_2D[camera_view, frame_idx, :, :],\n",
    "        Predictions_2D[camera_view, frame_idx, :, :],\n",
    "        joint_connections,\n",
    ")\n",
    "else:\n",
    "    fig = mv_nrsfm_kernel.get_figure2d(Predictions_2D[camera_view, frame_idx, :, :], GT_2D[camera_view, frame_idx, :, :])\n",
    "    py.iplot(fig)\n",
    "    \n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('mbw')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b582278b6a77aa0806905639f9b1b297c1a594406fb8e0b60c9f0b707f15f064"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}