code/09.UpdateSamples/Aug/mlx-phi35-vision.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_keyframes(video_path, output_folder):\n",
    "    videoCapture = cv2.VideoCapture(video_path)\n",
    "    success, frame = videoCapture.read()\n",
    "    i = 0\n",
    "    while success:\n",
    "        gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "        \n",
    "        hist = cv2.calcHist([gray_frame], [0], None, [256], [0, 256])\n",
    "        \n",
    "        success, next_frame = videoCapture.read()\n",
    "        if not success:\n",
    "            break\n",
    "        \n",
    "        next_gray_frame = cv2.cvtColor(next_frame, cv2.COLOR_BGR2GRAY)\n",
    "        \n",
    "        next_hist = cv2.calcHist([next_gray_frame], [0], None, [256], [0, 256])\n",
    "        \n",
    "        similarity = cv2.compareHist(hist, next_hist, cv2.HISTCMP_CORREL)\n",
    "        \n",
    "        if similarity < 0.9:\n",
    "            i += 1\n",
    "            cv2.imwrite(f\"{output_folder}/keyframe_{i}.jpg\", frame)\n",
    "            print(f\"Saved keyframe {i}\")\n",
    "        \n",
    "        frame = next_frame\n",
    "\n",
    "    videoCapture.release()\n",
    "    return i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved keyframe 1\n",
      "Saved keyframe 2\n",
      "Saved keyframe 3\n",
      "Saved keyframe 4\n",
      "Saved keyframe 5\n",
      "Saved keyframe 6\n",
      "Saved keyframe 7\n",
      "Saved keyframe 8\n",
      "Saved keyframe 9\n",
      "Saved keyframe 10\n",
      "Saved keyframe 11\n",
      "Saved keyframe 12\n"
     ]
    }
   ],
   "source": [
    "num = save_keyframes('./video/car.mp4', './output')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import requests, base64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = [] \n",
    "placeholder = \"\" \n",
    "for i in range(1,num+1): \n",
    "    with open(\"./output/keyframe_\"+str(i)+\".jpg\", \"rb\") as f:\n",
    "\n",
    "        images.append(Image.open(\"./output/keyframe_\"+str(i)+\".jpg\"))\n",
    "        placeholder += f\"<|image_{i}|>\\n\"\n",
    "        # print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mlx.core as mx\n",
    "from mlx_vlm import load, generate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "model_path = \"./phi-3.5-vision-mlx-int4\"\n",
    "model, processor = load(model_path,processor_config={\"trust_remote_code\":\"True\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [\n",
    "                {\"role\": \"user\", \"content\": \"Summarize the video.\"}, \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = processor.tokenizer.apply_chat_template(\n",
    "    messages,\n",
    "    tokenize=False,\n",
    "    add_generation_prompt=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>,\n",
       " <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x360>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = generate(model, processor, images, placeholder+prompt, verbose=False, max_tokens=1024)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"The video showcases a red Ferrari B12 Superfast car, highlighting its sleek design and luxurious features. The car is displayed from various angles, emphasizing its aerodynamic shape and the Ferrari logo on the front grille. The video also includes a close-up of the car's interior, showcasing the leather seats and the modern dashboard. The car is then seen driving on a winding road, demonstrating its performance capabilities.<|end|>\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mlxenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}