convert_images.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import gzip\n",
    "import pathlib\n",
    "import struct\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "from PIL import Image\n",
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def donwload(urls, path):\n",
    "    path.mkdir(parents=True, exist_ok=True)\n",
    "    for url in urls:\n",
    "        filepath = path / pathlib.Path(url).name\n",
    "        if not filepath.exists():\n",
    "            res = requests.get(url)\n",
    "            if res.status_code == 200:\n",
    "                with open(filepath, 'wb') as f:\n",
    "                    f.write(res.content)\n",
    "\n",
    "\n",
    "def load(paths):\n",
    "    x_path, y_path = paths\n",
    "    with gzip.open(x_path) as fx, gzip.open(y_path) as fy:\n",
    "        fx.read(4)\n",
    "        fy.read(4)\n",
    "        N, = struct.unpack('>i', fy.read(4))\n",
    "        if N != struct.unpack('>i', fx.read(4))[0]:\n",
    "            raise RuntimeError('wrong pair of MNIST images and labels')\n",
    "        fx.read(8)\n",
    "\n",
    "        images = np.empty((N, 784), dtype=np.uint8)\n",
    "        labels = np.empty(N, dtype=np.uint8)\n",
    "\n",
    "        for i in  tqdm(range(N)):\n",
    "            labels[i] = ord(fy.read(1))\n",
    "            for j in range(784):\n",
    "                images[i, j] = ord(fx.read(1))\n",
    "    return images, labels\n",
    "\n",
    "\n",
    "def make_images(path, kind, images, labels):\n",
    "    path.mkdir(parents=True, exist_ok=True)\n",
    "    for (i, image), label in zip(enumerate(images), labels):\n",
    "        filepath = path / '{}_{}_{}.jpg'.format(kind, label, i)\n",
    "        Image.fromarray(image.reshape(28, 28)).save(filepath)\n",
    "\n",
    "\n",
    "def make_labellist(path, kind, labels):\n",
    "    path.mkdir(parents=True, exist_ok=True)\n",
    "    filepaths = [\n",
    "        '{}_{}_{}.jpg'.format(kind,label, i) for i, label in enumerate(labels)\n",
    "    ]\n",
    "    df = pd.DataFrame({'name': filepaths, 'target': labels.tolist()})\n",
    "    df.to_csv(path / '{}.csv'.format(kind), index=False, header=False)\n",
    "\n",
    "\n",
    "def main(path='./src/data', out=\"jpg\"):\n",
    "    path = pathlib.Path(path)\n",
    "\n",
    "    def pipeline(kind, path):\n",
    "        _kind = kind\n",
    "        if kind == 'test':\n",
    "            _kind = 't10k'\n",
    "\n",
    "        baseurl = 'http://yann.lecun.com/exdb/mnist'\n",
    "        urls = [\n",
    "            '{}/{}-images-idx3-ubyte.gz'.format(baseurl, _kind),\n",
    "            '{}/{}-labels-idx1-ubyte.gz'.format(baseurl, _kind)\n",
    "        ]\n",
    "        donwload(urls, path / 'raw')\n",
    "\n",
    "        paths = [\n",
    "            path / 'raw' / '{}-images-idx3-ubyte.gz'.format(_kind),\n",
    "            path / 'raw' / '{}-labels-idx1-ubyte.gz'.format(_kind)\n",
    "        ]\n",
    "        images, labels = load(paths)\n",
    "\n",
    "        if out == 'jpg':\n",
    "            path = path / 'mnist'\n",
    "            make_images(path / 'images', kind, images, labels)\n",
    "            make_labellist(path / 'labels', kind, labels)\n",
    "        else:\n",
    "            path = path / 'mnist' / 'npz'\n",
    "            path.mkdir(parents=True, exist_ok=True)\n",
    "            np.savez_compressed(\n",
    "                path / '{}.npz'.format(kind), x=images, y=labels)\n",
    "\n",
    "    #print('Processing train data ...')\n",
    "    #pipeline('train', path)\n",
    "\n",
    "    print('Processing test data ...')\n",
    "    pipeline('test', path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! /Users/tdual/Downloads/t10k-images-idx3-ubyte.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! /Users/tdual/Downloads/t10k-labels-idx1-ubyte.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_list = [\n",
    "\"T-shirt_top\",\n",
    "\"Trouser\",\n",
    "\"Pullover\",\n",
    "\"Dress\",\n",
    "\"Coat\",\n",
    "\"Sandal\",\n",
    "\"Shirt\",\n",
    "\"Sneaker\",\n",
    "\"Bag\",\n",
    "\"Ankle boot\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ja_label_list = [\n",
    "\"Tシャツ・トップ\",\n",
    "\"ズボン\",\n",
    "\"プルオーバー\",\n",
    "\"ドレス\",\n",
    "\"コート\",\n",
    "\"サンダル\",\n",
    "\"シャツ\",\n",
    "\"スニーカー\",\n",
    "\"バッグ\",\n",
    "\"ブーツ\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_images(path, images, labels):\n",
    "    path.mkdir(parents=True, exist_ok=True)\n",
    "    for (i, image), label in zip(enumerate(images), labels):\n",
    "        filepath = path / '{}_{}.jpg'.format(label, i)\n",
    "        Image.fromarray(image.reshape(28, 28)).save(filepath)\n",
    "\n",
    "\n",
    "def make_labellist(path, labels):\n",
    "    path.mkdir(parents=True, exist_ok=True)\n",
    "    filepaths = [\n",
    "        '{}_{}.jpg'.format(label, i) for i, label in enumerate(labels)\n",
    "    ]\n",
    "    df = pd.DataFrame({'name': filepaths, 'target': labels})\n",
    "    df.to_csv(path / 'labels.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(paths):\n",
    "    x_path, y_path = paths\n",
    "    with gzip.open(x_path) as fx, gzip.open(y_path) as fy:\n",
    "        fx.read(4)\n",
    "        fy.read(4)\n",
    "        N, = struct.unpack('>i', fy.read(4))\n",
    "        if N != struct.unpack('>i', fx.read(4))[0]:\n",
    "            raise RuntimeError('wrong pair of MNIST images and labels')\n",
    "        fx.read(8)\n",
    "\n",
    "        images = np.empty((N, 784), dtype=np.uint8)\n",
    "        #labels = np.empty(N, dtype=np.uint8)\n",
    "\n",
    "        labels = []\n",
    "        \n",
    "        for i in  tqdm(range(N)):\n",
    "            labels.append(ja_label_list[ord(fy.read(1))])\n",
    "            for j in range(784):\n",
    "                images[i, j] = ord(fx.read(1))\n",
    "    return images, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths = [\n",
    "    \"/Users/tdual/Downloads/train-images-idx3-ubyte.gz\",\n",
    "    \"/Users/tdual/Downloads/train-labels-idx1-ubyte.gz\"\n",
    "]\n",
    "path = pathlib.Path(\"/Users/tdual/Downloads/fashion_train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e700ef8005943f7a255a5a4b7686a47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=60000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "['ブーツ', 'Tシャツ・トップ', 'Tシャツ・トップ', 'ドレス', 'Tシャツ・トップ', 'プルオーバー', 'スニーカー', 'プルオーバー', 'サンダル', 'サンダル']\n"
     ]
    }
   ],
   "source": [
    "images, labels = load(paths)\n",
    "print(labels[:10])\n",
    "make_images(path / 'images', images, labels)\n",
    "make_labellist(path / 'labels', labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_py3.6)",
   "language": "python",
   "name": "conda_py3.6"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}