chp02_Datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "caf18729-6c15-4ec3-8bdc-f0c0b8844ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import urllib.request\n",
    "import zipfile\n",
    "\n",
    "import warnings\n",
    "\n",
    "\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9dcbcc4c-4e4f-44c4-8181-8b5d64d7e442",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_url = 'https://github.com/mattharrison/datasets/raw/master/data/kaggle-survey-2018.zip'\n",
    "file_name = 'kaggle-survey-2018.zip'\n",
    "dataset = 'multipleChoiceResponses.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8689f736-7047-40cf-863c-12358068416d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_dataset(src_path: str, dst_path: str, dataset: str) -> pd.DataFrame: \n",
    "    \"\"\"Extracts a chosen dataset from a zip file and read it into a pandas\n",
    "    DataFrame.\n",
    "    \n",
    "    Parameters:\n",
    "    ------------\n",
    "    src_path: str\n",
    "        URL or path of the zip file to be downloaded.\n",
    "    dst_path: str\n",
    "        Local file path where the zip file will be written.\n",
    "    dataset: str\n",
    "        Name of the particular file inside the zip file to be \n",
    "        read into a DataFrame.\n",
    "    \n",
    "    Returns:\n",
    "    ---------\n",
    "    pandas.core.frame.DataFrame: \n",
    "        DataFrame containing the contents of the selected dataset.\n",
    "        \n",
    "    \"\"\"\n",
    "    \n",
    "    # using `urllib.request` module to open the URL specified in `src_path`.    \n",
    "    open_path = urllib.request.urlopen(src_path)\n",
    "    \n",
    "    # reads the contents of the file object `open_path`\n",
    "    data = open_path.read()\n",
    "    \n",
    "    # using context management to write the file in the destination path or folder\n",
    "    with open(dst_path, mode='wb') as f:\n",
    "        f.write(data)\n",
    "       \n",
    "    # using context management to extract the saved zip file.\n",
    "    with zipfile.ZipFile(dst_path) as z:\n",
    "        all_df = pd.read_csv(z.open(dataset))\n",
    "        df_ = all_df.iloc[0]\n",
    "        df = all_df.iloc[1:]\n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4e9bdb5e-4845-4f4f-ab62-2cadf328f7e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ricky\\AppData\\Local\\Temp\\ipykernel_2820\\2718579718.py:34: DtypeWarning: Columns (0,2,8,10,21,23,24,25,26,27,28,44,56,64,83,85,87,107,109,123,125,150,157,172,174,194,210,218,219,223,246,249,262,264,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,304,306,325,326,329,341,368,371,384,385,389,390,391,393,394) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  all_df = pd.read_csv(z.open(dataset))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time from Start to Finish (seconds)</th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q1_OTHER_TEXT</th>\n",
       "      <th>Q2</th>\n",
       "      <th>Q3</th>\n",
       "      <th>Q4</th>\n",
       "      <th>Q5</th>\n",
       "      <th>Q6</th>\n",
       "      <th>Q6_OTHER_TEXT</th>\n",
       "      <th>Q7</th>\n",
       "      <th>...</th>\n",
       "      <th>Q49_OTHER_TEXT</th>\n",
       "      <th>Q50_Part_1</th>\n",
       "      <th>Q50_Part_2</th>\n",
       "      <th>Q50_Part_3</th>\n",
       "      <th>Q50_Part_4</th>\n",
       "      <th>Q50_Part_5</th>\n",
       "      <th>Q50_Part_6</th>\n",
       "      <th>Q50_Part_7</th>\n",
       "      <th>Q50_Part_8</th>\n",
       "      <th>Q50_OTHER_TEXT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9666</th>\n",
       "      <td>4964</td>\n",
       "      <td>Male</td>\n",
       "      <td>-1</td>\n",
       "      <td>22-24</td>\n",
       "      <td>India</td>\n",
       "      <td>Bachelor’s degree</td>\n",
       "      <td>Computer science (software engineering, etc.)</td>\n",
       "      <td>Student</td>\n",
       "      <td>-1</td>\n",
       "      <td>I am a student</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Too time-consuming</td>\n",
       "      <td>Requires too much technical knowledge</td>\n",
       "      <td>Afraid that others will use my work without gi...</td>\n",
       "      <td>Not enough incentives to share my work</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3604</th>\n",
       "      <td>467</td>\n",
       "      <td>Female</td>\n",
       "      <td>-1</td>\n",
       "      <td>25-29</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>Master’s degree</td>\n",
       "      <td>Medical or life sciences (biology, chemistry, ...</td>\n",
       "      <td>Statistician</td>\n",
       "      <td>-1</td>\n",
       "      <td>Academics/Education</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Afraid that others will use my work without gi...</td>\n",
       "      <td>Not enough incentives to share my work</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7309</th>\n",
       "      <td>49085</td>\n",
       "      <td>Male</td>\n",
       "      <td>-1</td>\n",
       "      <td>45-49</td>\n",
       "      <td>South Africa</td>\n",
       "      <td>Bachelor’s degree</td>\n",
       "      <td>Engineering (non-computer focused)</td>\n",
       "      <td>Other</td>\n",
       "      <td>349</td>\n",
       "      <td>Other</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None of these reasons apply to me</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23183</th>\n",
       "      <td>759</td>\n",
       "      <td>Male</td>\n",
       "      <td>-1</td>\n",
       "      <td>25-29</td>\n",
       "      <td>Germany</td>\n",
       "      <td>Master’s degree</td>\n",
       "      <td>Engineering (non-computer focused)</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>-1</td>\n",
       "      <td>Marketing/CRM</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7293</th>\n",
       "      <td>1123</td>\n",
       "      <td>Male</td>\n",
       "      <td>-1</td>\n",
       "      <td>40-44</td>\n",
       "      <td>Chile</td>\n",
       "      <td>Doctoral degree</td>\n",
       "      <td>Engineering (non-computer focused)</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>-1</td>\n",
       "      <td>Broadcasting/Communications</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>Too expensive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4418</th>\n",
       "      <td>507</td>\n",
       "      <td>Male</td>\n",
       "      <td>-1</td>\n",
       "      <td>22-24</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>Master’s degree</td>\n",
       "      <td>Computer science (software engineering, etc.)</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>-1</td>\n",
       "      <td>Insurance/Risk Assessment</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Too time-consuming</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 395 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Time from Start to Finish (seconds)      Q1 Q1_OTHER_TEXT     Q2  \\\n",
       "9666                                 4964    Male            -1  22-24   \n",
       "3604                                  467  Female            -1  25-29   \n",
       "7309                                49085    Male            -1  45-49   \n",
       "23183                                 759    Male            -1  25-29   \n",
       "7293                                 1123    Male            -1  40-44   \n",
       "4418                                  507    Male            -1  22-24   \n",
       "\n",
       "                             Q3                 Q4  \\\n",
       "9666                      India  Bachelor’s degree   \n",
       "3604   United States of America    Master’s degree   \n",
       "7309               South Africa  Bachelor’s degree   \n",
       "23183                   Germany    Master’s degree   \n",
       "7293                      Chile    Doctoral degree   \n",
       "4418   United States of America    Master’s degree   \n",
       "\n",
       "                                                      Q5              Q6  \\\n",
       "9666       Computer science (software engineering, etc.)         Student   \n",
       "3604   Medical or life sciences (biology, chemistry, ...    Statistician   \n",
       "7309                  Engineering (non-computer focused)           Other   \n",
       "23183                 Engineering (non-computer focused)  Data Scientist   \n",
       "7293                  Engineering (non-computer focused)  Data Scientist   \n",
       "4418       Computer science (software engineering, etc.)  Data Scientist   \n",
       "\n",
       "      Q6_OTHER_TEXT                           Q7  ... Q49_OTHER_TEXT  \\\n",
       "9666             -1               I am a student  ...             -1   \n",
       "3604             -1          Academics/Education  ...             -1   \n",
       "7309            349                        Other  ...             -1   \n",
       "23183            -1                Marketing/CRM  ...             -1   \n",
       "7293             -1  Broadcasting/Communications  ...             -1   \n",
       "4418             -1    Insurance/Risk Assessment  ...             -1   \n",
       "\n",
       "          Q50_Part_1          Q50_Part_2  \\\n",
       "9666             NaN  Too time-consuming   \n",
       "3604             NaN                 NaN   \n",
       "7309             NaN                 NaN   \n",
       "23183            NaN                 NaN   \n",
       "7293   Too expensive                 NaN   \n",
       "4418             NaN  Too time-consuming   \n",
       "\n",
       "                                  Q50_Part_3  \\\n",
       "9666   Requires too much technical knowledge   \n",
       "3604                                     NaN   \n",
       "7309                                     NaN   \n",
       "23183                                    NaN   \n",
       "7293                                     NaN   \n",
       "4418                                     NaN   \n",
       "\n",
       "                                              Q50_Part_4  \\\n",
       "9666   Afraid that others will use my work without gi...   \n",
       "3604   Afraid that others will use my work without gi...   \n",
       "7309                                                 NaN   \n",
       "23183                                                NaN   \n",
       "7293                                                 NaN   \n",
       "4418                                                 NaN   \n",
       "\n",
       "                                   Q50_Part_5 Q50_Part_6  \\\n",
       "9666   Not enough incentives to share my work        NaN   \n",
       "3604   Not enough incentives to share my work        NaN   \n",
       "7309                                      NaN        NaN   \n",
       "23183                                     NaN        NaN   \n",
       "7293                                      NaN        NaN   \n",
       "4418                                      NaN        NaN   \n",
       "\n",
       "                              Q50_Part_7 Q50_Part_8 Q50_OTHER_TEXT  \n",
       "9666                                 NaN        NaN             -1  \n",
       "3604                                 NaN        NaN             -1  \n",
       "7309   None of these reasons apply to me        NaN             -1  \n",
       "23183                                NaN        NaN             -1  \n",
       "7293                                 NaN        NaN             -1  \n",
       "4418                                 NaN        NaN             -1  \n",
       "\n",
       "[6 rows x 395 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "survey_df = extract_dataset(path_url, file_name, dataset)\n",
    "survey_df.sample(6)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45d72f6a-0d3e-462e-b5af-98551b8165fd",
   "metadata": {},
   "source": [
    "### Pull off the first two characters of the Q2 column and convert them to an integer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "149404e0-fd95-4a7f-a45a-0d8bdcbd6e64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        45\n",
       "2        30\n",
       "3        30\n",
       "4        35\n",
       "5        22\n",
       "         ..\n",
       "23855    45\n",
       "23856    25\n",
       "23857    22\n",
       "23858    25\n",
       "23859    25\n",
       "Name: Q2, Length: 23859, dtype: int32"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df['Q2']\n",
    " .str.slice(0,2)\n",
    " .astype(int)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c1db3fa-af7d-4aea-a90f-3cef599ea0f4",
   "metadata": {},
   "source": [
    "### Replace the education strings with numeric values.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1da3e859-0772-4044-8579-9626a1a36c98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Doctoral degree', 'Bachelor’s degree', 'Master’s degree',\n",
       "       'Professional degree',\n",
       "       'Some college/university study without earning a bachelor’s degree',\n",
       "       'I prefer not to answer', 'No formal education past high school',\n",
       "       nan], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q4']\n",
    " .unique()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7e35b450-4f78-4a3a-a296-dbdaa07ba8e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        20.0\n",
       "2        16.0\n",
       "3        18.0\n",
       "4        18.0\n",
       "5        18.0\n",
       "         ... \n",
       "23855    20.0\n",
       "23856    18.0\n",
       "23857    18.0\n",
       "23858     NaN\n",
       "23859    18.0\n",
       "Name: Q4, Length: 23859, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q4']\n",
    " .replace({'Doctoral degree' : 20, 'Bachelor’s degree' : 16, 'Master’s degree' : 18,\n",
    "       'Professional degree' : 19,\n",
    "       'Some college/university study without earning a bachelor’s degree': 13,\n",
    "       'I prefer not to answer' : None, 'No formal education past high school': 12})\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e593f4f-e7dc-4069-8230-deab8534d43d",
   "metadata": {},
   "source": [
    "### Take the top three majors, change the others to 'other'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b04adca0-c751-44c6-b4f8-b5ba7abc4f8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1                                                    Other\n",
       "2                       Engineering (non-computer focused)\n",
       "3            Computer science (software engineering, etc.)\n",
       "4        Social sciences (anthropology, psychology, soc...\n",
       "5                                Mathematics or statistics\n",
       "                               ...                        \n",
       "23855        Computer science (software engineering, etc.)\n",
       "23856                   Engineering (non-computer focused)\n",
       "23857        Computer science (software engineering, etc.)\n",
       "23858                                                  NaN\n",
       "23859                            Mathematics or statistics\n",
       "Name: Q5, Length: 23859, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q5']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2c5a3031-9f9f-4471-a1f3-b665a7d868fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Computer science (software engineering, etc.)                    9430\n",
       "Engineering (non-computer focused)                               3705\n",
       "Mathematics or statistics                                        2950\n",
       "A business discipline (accounting, economics, finance, etc.)     1791\n",
       "Physics or astronomy                                             1110\n",
       "Information technology, networking, or system administration     1029\n",
       "Medical or life sciences (biology, chemistry, medicine, etc.)     871\n",
       "Other                                                             770\n",
       "Social sciences (anthropology, psychology, sociology, etc.)       554\n",
       "Humanities (history, literature, philosophy, etc.)                269\n",
       "Environmental science or geology                                  253\n",
       "I never declared a major                                          128\n",
       "Fine arts or performing arts                                       87\n",
       "Name: Q5, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q5']\n",
    " .value_counts()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "648e8ae4-ae98-43e1-bccd-ddbb214fcd4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1                                                other\n",
       "2                   Engineering (non-computer focused)\n",
       "3        Computer science (software engineering, etc.)\n",
       "4                                                other\n",
       "5                            Mathematics or statistics\n",
       "                             ...                      \n",
       "23855    Computer science (software engineering, etc.)\n",
       "23856               Engineering (non-computer focused)\n",
       "23857    Computer science (software engineering, etc.)\n",
       "23858                                            other\n",
       "23859                        Mathematics or statistics\n",
       "Name: Q5, Length: 23859, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = (survey_df\n",
    " ['Q5']\n",
    " .value_counts()\n",
    ")\n",
    "\n",
    "(survey_df\n",
    " ['Q5']\n",
    " .where(survey_df['Q5'].isin(count.index[:4]), 'other')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b6b885a-676b-4ca1-b287-dec9ac1ebdb3",
   "metadata": {},
   "source": [
    "### convert to a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0423e1f8-34e9-484d-839c-2d411c48ac02",
   "metadata": {},
   "outputs": [],
   "source": [
    "def topx(pser: pd.Series, x: int = 5, default='other') -> pd.Series:\n",
    "    \"\"\"Replace all values in a Pandas Series that are not among\n",
    "    the top `x` most frequent values with a default value.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    pser : pd.Series\n",
    "        The input pandas Series.\n",
    "    x : int, optional\n",
    "        The number of most frequent values to keep. The\n",
    "        default value is 5.\n",
    "    default : str, optional\n",
    "        The default value to use for values that are not among\n",
    "        the top `x` most frequent values. The default value is\n",
    "        'other'.\n",
    "        \n",
    "    Returns:\n",
    "    ---------\n",
    "    pandas.Series: \n",
    "        Series containing the results.\n",
    "    \"\"\"\n",
    "    \n",
    "    val_count = pser.value_counts()\n",
    "    return (pser.where(pser\n",
    "                      .isin(val_count.index[:x]),\n",
    "                      default)\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3aa301c8-9ea1-4fb6-a864-6572022b3c2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1                                                other\n",
       "2                   Engineering (non-computer focused)\n",
       "3        Computer science (software engineering, etc.)\n",
       "4                                                other\n",
       "5                            Mathematics or statistics\n",
       "                             ...                      \n",
       "23855    Computer science (software engineering, etc.)\n",
       "23856               Engineering (non-computer focused)\n",
       "23857    Computer science (software engineering, etc.)\n",
       "23858                                            other\n",
       "23859                        Mathematics or statistics\n",
       "Name: Q5, Length: 23859, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topx(survey_df['Q5'],)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "511c116a-9181-422a-8ea1-fe00c3684d2e",
   "metadata": {},
   "source": [
    "### rename those top three majors to shortened versions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6067c184-33a7-4e52-b9a8-efcac666dd7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['other', 'Engineering (non-computer focused)',\n",
       "       'Computer science (software engineering, etc.)',\n",
       "       'Mathematics or statistics', 'Physics or astronomy',\n",
       "       'A business discipline (accounting, economics, finance, etc.)'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topx(survey_df['Q5'],).unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "31182bce-3045-4e3f-b135-e329cb0ad589",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        other\n",
       "2          eng\n",
       "3           cs\n",
       "4        other\n",
       "5         stat\n",
       "         ...  \n",
       "23855       cs\n",
       "23856      eng\n",
       "23857       cs\n",
       "23858    other\n",
       "23859     stat\n",
       "Name: Q5, Length: 23859, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(topx(survey_df['Q5'],)\n",
    " .replace({'Engineering (non-computer focused)': 'eng',\n",
    "       'Computer science (software engineering, etc.)': 'cs',\n",
    "       'Mathematics or statistics' : 'stat',})\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcefe1d0-d63b-469b-9650-2f6464d8a61b",
   "metadata": {},
   "source": [
    "### Convert the Q8 column to years of experience "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ba97ff8b-58cf-4a80-8382-d15e2fb709f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '5-10', '0-1', '10-15', '3-4', '1-2', '2-3', '15-20', '4-5',\n",
       "       '20-25', '25-30', '30 +'], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q8']\n",
    " # .str.replace('+', '', regex=False)\n",
    ").unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b2773ab-9412-46d2-b375-060416410227",
   "metadata": {},
   "source": [
    "From the above output, these would be my steps:\n",
    "\n",
    "* replace '+' empty space, \n",
    "* split on '-' (the first value of the range) \n",
    "* take the left-hand side\n",
    "* converte that value to a floating point number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "13802b69-4489-407d-90fc-ba2c102c113c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        NaN\n",
       "2        5.0\n",
       "3        0.0\n",
       "4        NaN\n",
       "5        0.0\n",
       "        ... \n",
       "23855    5.0\n",
       "23856    NaN\n",
       "23857    0.0\n",
       "23858    NaN\n",
       "23859    NaN\n",
       "Name: 0, Length: 23859, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q8']\n",
    " .str.replace('+', '', regex=False)\n",
    " .str.split('-', expand=True)\n",
    " .iloc[:,0]\n",
    " .astype(float)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17b5c96e-4225-4da0-a989-b62bb921818c",
   "metadata": {},
   "source": [
    "### Replace values in the Q9 column "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "789c1b46-28ac-4b05-b807-0686ac3d3734",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '10-20,000', '0-10,000',\n",
       "       'I do not wish to disclose my approximate yearly compensation',\n",
       "       '20-30,000', '125-150,000', '30-40,000', '50-60,000',\n",
       "       '100-125,000', '90-100,000', '70-80,000', '80-90,000', '60-70,000',\n",
       "       '400-500,000', '40-50,000', '150-200,000', '500,000+',\n",
       "       '300-400,000', '200-250,000', '250-300,000'], dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q9']\n",
    " .unique()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a64dedf7-f20d-4ec9-8a50-4dda949b3045",
   "metadata": {},
   "source": [
    "* remove `+` & `,`\n",
    "* shorten 500,000 to 500, \n",
    "* replace the long string with '0'\n",
    "* split on '-' (the first value of the range) and take the left side, \n",
    "* fill in missing values with zero, \n",
    "* convert the values to an integer \n",
    "* multiply it by 1,000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f6dab882-094e-460f-9f5e-8d89d188b630",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1             0\n",
       "2         10000\n",
       "3             0\n",
       "4             0\n",
       "5             0\n",
       "          ...  \n",
       "23855    250000\n",
       "23856         0\n",
       "23857     10000\n",
       "23858         0\n",
       "23859         0\n",
       "Name: 0, Length: 23859, dtype: int32"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q9']\n",
    " .str.replace('+', '', regex=False)\n",
    " .str.replace(',','')\n",
    " .replace({'500000':'500',\n",
    "          'I do not wish to disclose my approximate yearly compensation':'0'})\n",
    " .str.split('-', expand=True)\n",
    " .iloc[:,0]\n",
    " .fillna(0)\n",
    " .astype(int)\n",
    " .mul(1_000)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8535998-8a5e-4da5-aace-9ecb94362e84",
   "metadata": {},
   "source": [
    "### Final clean up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6372bc32-4ea0-4306-8607-deda4134d16f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'Python'], dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q16_Part_1']\n",
    ").unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e06e1347-26d3-4779-b907-2e62f93d85c0",
   "metadata": {},
   "source": [
    "* Fill in missing values of `Q16_Part_1` with zero \n",
    "* Replace the value `Python` with 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1194bee0-4200-4b01-9d81-809d10126265",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        0\n",
       "2        0\n",
       "3        0\n",
       "4        1\n",
       "5        0\n",
       "        ..\n",
       "23855    1\n",
       "23856    0\n",
       "23857    0\n",
       "23858    0\n",
       "23859    0\n",
       "Name: Q16_Part_1, Length: 23859, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q16_Part_1']\n",
    " .fillna(0)\n",
    " .replace({'Python': 1})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b1378ada-4051-4e35-b3dc-25b810ada161",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'R'], dtype=object)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q16_Part_2']\n",
    ").unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ef03e8a-1178-452b-aec4-f5199d0798c8",
   "metadata": {},
   "source": [
    "* Fill in missing values of `Q16_Part_2` with 0\n",
    "* Replace the value `R` with 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7de0447f-d723-47d9-99c7-6e39a3f9a059",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        0\n",
       "2        0\n",
       "3        1\n",
       "4        1\n",
       "5        0\n",
       "        ..\n",
       "23855    1\n",
       "23856    0\n",
       "23857    0\n",
       "23858    0\n",
       "23859    0\n",
       "Name: Q16_Part_2, Length: 23859, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q16_Part_2']\n",
    " .fillna(0)\n",
    " .replace({'R': 1})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "261360fc-6a54-412e-a34a-133f5fed3e2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'SQL'], dtype=object)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q16_Part_3']\n",
    ").unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd9fc6ac-6cd3-4580-a56f-36d9b6734851",
   "metadata": {},
   "source": [
    "* Fill in missing values of `Q16_Part_3` with 0\n",
    "* Replace the value `SQL` with 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "54241262-0339-4cfa-aedf-118ee46c9a30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        0\n",
       "2        1\n",
       "3        0\n",
       "4        1\n",
       "5        1\n",
       "        ..\n",
       "23855    1\n",
       "23856    0\n",
       "23857    0\n",
       "23858    0\n",
       "23859    0\n",
       "Name: Q16_Part_3, Length: 23859, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(survey_df\n",
    " ['Q16_Part_3']\n",
    " .fillna(0)\n",
    " .replace({'SQL': 1})\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94be624a-fc77-4a34-815c-ef720200d568",
   "metadata": {},
   "source": [
    "### Create a function\n",
    "*  rename the columns by replacing spaces with an underscore.\n",
    "* pull out only the Q1, Q2, age, education, major, years_exp, compensation, python, r, and sql columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bdd5e86a-c6ef-4558-afa3-83780eae0fa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Q1',\n",
       " 'Q3',\n",
       " 'age',\n",
       " 'education',\n",
       " 'major',\n",
       " 'years_exp',\n",
       " 'compensation',\n",
       " 'python',\n",
       " 'r',\n",
       " 'sql']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'Q1,Q3,age,education,major,years_exp,compensation,python,r,sql'.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6304021b-a2ce-4a34-8f15-27531c94479e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q3</th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>major</th>\n",
       "      <th>years_exp</th>\n",
       "      <th>compensation</th>\n",
       "      <th>python</th>\n",
       "      <th>r</th>\n",
       "      <th>sql</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Female</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>45</td>\n",
       "      <td>20.0</td>\n",
       "      <td>other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>Indonesia</td>\n",
       "      <td>30</td>\n",
       "      <td>16.0</td>\n",
       "      <td>eng</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Female</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>30</td>\n",
       "      <td>18.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Male</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>35</td>\n",
       "      <td>18.0</td>\n",
       "      <td>other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Male</td>\n",
       "      <td>India</td>\n",
       "      <td>22</td>\n",
       "      <td>18.0</td>\n",
       "      <td>stat</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23855</th>\n",
       "      <td>Male</td>\n",
       "      <td>France</td>\n",
       "      <td>45</td>\n",
       "      <td>20.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>5.0</td>\n",
       "      <td>250000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23856</th>\n",
       "      <td>Female</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>25</td>\n",
       "      <td>18.0</td>\n",
       "      <td>eng</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23857</th>\n",
       "      <td>Male</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>22</td>\n",
       "      <td>18.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23858</th>\n",
       "      <td>Male</td>\n",
       "      <td>United Kingdom of Great Britain and Northern I...</td>\n",
       "      <td>25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23859</th>\n",
       "      <td>Male</td>\n",
       "      <td>Spain</td>\n",
       "      <td>25</td>\n",
       "      <td>18.0</td>\n",
       "      <td>stat</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23859 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Q1                                                 Q3  age  \\\n",
       "1      Female                           United States of America   45   \n",
       "2        Male                                          Indonesia   30   \n",
       "3      Female                           United States of America   30   \n",
       "4        Male                           United States of America   35   \n",
       "5        Male                                              India   22   \n",
       "...       ...                                                ...  ...   \n",
       "23855    Male                                             France   45   \n",
       "23856  Female                                             Turkey   25   \n",
       "23857    Male                                             Turkey   22   \n",
       "23858    Male  United Kingdom of Great Britain and Northern I...   25   \n",
       "23859    Male                                              Spain   25   \n",
       "\n",
       "       education  major  years_exp  compensation  python  r  sql  \n",
       "1           20.0  other        NaN             0       0  0    0  \n",
       "2           16.0    eng        5.0         10000       0  0    1  \n",
       "3           18.0     cs        0.0             0       0  1    0  \n",
       "4           18.0  other        NaN             0       1  1    1  \n",
       "5           18.0   stat        0.0             0       0  0    1  \n",
       "...          ...    ...        ...           ...     ... ..  ...  \n",
       "23855       20.0     cs        5.0        250000       1  1    1  \n",
       "23856       18.0    eng        NaN             0       0  0    0  \n",
       "23857       18.0     cs        0.0         10000       0  0    0  \n",
       "23858        NaN  other        NaN             0       0  0    0  \n",
       "23859       18.0   stat        NaN             0       0  0    0  \n",
       "\n",
       "[23859 rows x 10 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def prep_data(pdf: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\" This function prepares the rurvey data and returns a new DataFrame.\n",
    "    The prepatations include extracting and transforming certain\n",
    "    columns, renaming some, and selecting a subset of columns.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    pdf : pd.DataFrame\n",
    "        The input DataFrame containing the survey data.\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame:\n",
    "        The new DataFrame with the modified and selected columns.\n",
    "    \"\"\"\n",
    "    \n",
    "    return (pdf\n",
    "            .assign(age=(pdf['Q2']\n",
    "                         .str.slice(0,2)\n",
    "                         .astype(int)\n",
    "                        ),\n",
    "                   education=(survey_df\n",
    "                             ['Q4']\n",
    "                             .replace({'Doctoral degree' : 20, 'Bachelor’s degree' : 16, \n",
    "                                       'Master’s degree' : 18,\n",
    "                                   'Professional degree' : 19,\n",
    "                                   'Some college/university study without earning a bachelor’s degree': 13,\n",
    "                                   'I prefer not to answer' : None, \n",
    "                                    'No formal education past high school': 12})\n",
    "                            ),\n",
    "                    major=(pdf['Q5']\n",
    "                           .pipe(topx, x=3)\n",
    "                           .replace({'Engineering (non-computer focused)': 'eng',\n",
    "                                       'Computer science (software engineering, etc.)': 'cs',\n",
    "                                       'Mathematics or statistics' : 'stat',})\n",
    "                          ),\n",
    "                    years_exp=(pdf\n",
    "                             ['Q8']\n",
    "                             .str.replace('+', '', regex=False)\n",
    "                             .str.split('-', expand=True)\n",
    "                             .iloc[:,0]\n",
    "                             .astype(float)\n",
    "                            ),\n",
    "                    compensation=(pdf\n",
    "                             ['Q9']\n",
    "                             .str.replace('+', '', regex=False)\n",
    "                             .str.replace(',','')\n",
    "                             .replace({'500000':'500',\n",
    "                                      'I do not wish to disclose my approximate yearly compensation':'0'})\n",
    "                             .str.split('-', expand=True)\n",
    "                             .iloc[:,0]\n",
    "                             .fillna(0)\n",
    "                             .astype(int)\n",
    "                             .mul(1_000)\n",
    "                            ),\n",
    "                    python=(pdf\n",
    "                             ['Q16_Part_1']\n",
    "                             .fillna(0)\n",
    "                             .replace({'Python': 1})\n",
    "                            ),\n",
    "                    r=(pdf\n",
    "                         ['Q16_Part_2']\n",
    "                         .fillna(0)\n",
    "                         .replace({'R': 1})\n",
    "                        ),\n",
    "                    sql=(pdf\n",
    "                         ['Q16_Part_3']\n",
    "                         .fillna(0)\n",
    "                         .replace({'SQL': 1})\n",
    "                        )\n",
    "                    )\n",
    "            .rename(columns=lambda col:col.replace(' ', '_'))\n",
    "            .loc[:, 'Q1,Q3,age,education,major,years_exp,compensation,python,r,sql'.split(',')]\n",
    "    )\n",
    "\n",
    "final_df = prep_data(survey_df)\n",
    "final_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "898d34be-5cd1-4656-be1a-31f185fa2429",
   "metadata": {},
   "source": [
    "The transformer class subclasses the `BaseEstimator` and `TransformerMixin` classes. These\n",
    "classes require that we implement the `.fit` and `.transform` methods, respectively. The `.fit`\n",
    "method returns the class instance. The .transform method leverages the logic in the `prep_data`\n",
    "function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b9990e33-81d5-4a99-8bbc-e609daf491ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from feature_engine import encoding, imputation\n",
    "from sklearn import base, pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "37d90216-7648-4c00-9927-9df7d0ba8628",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrepDataTransformer(base.BaseEstimator,\n",
    "    base.TransformerMixin):\n",
    "    \"\"\"\n",
    "    This transformer takes a Pandas DataFrame containing our survey \n",
    "    data as input and returns a new version of the DataFrame. \n",
    "    \n",
    "    ----------\n",
    "    ycol : str, optional\n",
    "        The name of the column to be used as the target variable.\n",
    "        If not specified, the target variable will not be set.\n",
    "    Attributes\n",
    "    ----------\n",
    "    ycol : str\n",
    "        The name of the column to be used as the target variable.\n",
    "    \"\"\"\n",
    "    def __init__(self, ycol=None):\n",
    "        self.ycol = ycol\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return prep_data(X)\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "906a5c28-c036-4aa0-aa72-e7a21d6bc5e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepX_y(df, y_col):\n",
    "    raw = (df\n",
    "    .query('Q3.isin([\"United States of America\", \"China\", \"India\"]) '\n",
    "    'and Q6.isin([\"Data Scientist\", \"Software Engineer\"])')\n",
    "    )\n",
    "    return raw.drop(columns=[y_col]), raw[y_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "60c6406f-4b29-4a73-af40-213f463f8938",
   "metadata": {},
   "outputs": [],
   "source": [
    "pline = pipeline.Pipeline(\n",
    "    [('tweak', PrepDataTransformer()),\n",
    "    ('cat', encoding.OneHotEncoder(top_categories=5, drop_last=True,\n",
    "    variables=['Q1', 'Q3', 'major'])),\n",
    "    ('num_impute', imputation.MeanMedianImputer(imputation_method='median',\n",
    "                variables=['education', 'years_exp']))]\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "bb576500-a697-4484-9282-9af41cc7d3a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3836ff9b-6f46-40b2-b979-229b799bca51",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = prepX_y(survey_df, 'Q6')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "95de63a7-257f-4d03-9ad4-d6fc3a89e9cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "sur_X_train, sur_X_test, sur_y_train, sur_y_test = (model_selection\n",
    "                                    .train_test_split(X, y, \n",
    "                                                      test_size=.3, \n",
    "                                                      random_state=42, \n",
    "                                                      stratify=y)\n",
    "                                                        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8f5431c4-c176-46cc-87d8-6e4e057b7c97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>years_exp</th>\n",
       "      <th>compensation</th>\n",
       "      <th>python</th>\n",
       "      <th>r</th>\n",
       "      <th>sql</th>\n",
       "      <th>Q1_Male</th>\n",
       "      <th>Q1_Female</th>\n",
       "      <th>Q1_Prefer not to say</th>\n",
       "      <th>Q1_Prefer to self-describe</th>\n",
       "      <th>Q3_United States of America</th>\n",
       "      <th>Q3_India</th>\n",
       "      <th>Q3_China</th>\n",
       "      <th>major_cs</th>\n",
       "      <th>major_other</th>\n",
       "      <th>major_eng</th>\n",
       "      <th>major_stat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>25</td>\n",
       "      <td>18.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3065</th>\n",
       "      <td>22</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8435</th>\n",
       "      <td>22</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3110</th>\n",
       "      <td>40</td>\n",
       "      <td>20.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>125000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16372</th>\n",
       "      <td>45</td>\n",
       "      <td>12.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>100000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16608</th>\n",
       "      <td>25</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7325</th>\n",
       "      <td>18</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21810</th>\n",
       "      <td>18</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4917</th>\n",
       "      <td>25</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>100000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>639</th>\n",
       "      <td>25</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2110 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       age  education  years_exp  compensation  python  r  sql  Q1_Male  \\\n",
       "587     25       18.0        4.0             0       1  0    1        1   \n",
       "3065    22       16.0        1.0         10000       1  0    0        1   \n",
       "8435    22       18.0        1.0             0       1  0    0        1   \n",
       "3110    40       20.0        3.0        125000       1  0    1        0   \n",
       "16372   45       12.0        5.0        100000       1  0    1        1   \n",
       "...    ...        ...        ...           ...     ... ..  ...      ...   \n",
       "16608   25       16.0        2.0         10000       0  0    1        1   \n",
       "7325    18       16.0        1.0         30000       1  0    1        1   \n",
       "21810   18       16.0        2.0             0       0  0    0        1   \n",
       "4917    25       18.0        1.0        100000       1  0    1        1   \n",
       "639     25       18.0        1.0         10000       1  0    0        0   \n",
       "\n",
       "       Q1_Female  Q1_Prefer not to say  Q1_Prefer to self-describe  \\\n",
       "587            0                     0                           0   \n",
       "3065           0                     0                           0   \n",
       "8435           0                     0                           0   \n",
       "3110           1                     0                           0   \n",
       "16372          0                     0                           0   \n",
       "...          ...                   ...                         ...   \n",
       "16608          0                     0                           0   \n",
       "7325           0                     0                           0   \n",
       "21810          0                     0                           0   \n",
       "4917           0                     0                           0   \n",
       "639            1                     0                           0   \n",
       "\n",
       "       Q3_United States of America  Q3_India  Q3_China  major_cs  major_other  \\\n",
       "587                              0         1         0         0            1   \n",
       "3065                             0         1         0         1            0   \n",
       "8435                             0         1         0         0            1   \n",
       "3110                             1         0         0         0            1   \n",
       "16372                            1         0         0         0            1   \n",
       "...                            ...       ...       ...       ...          ...   \n",
       "16608                            0         1         0         1            0   \n",
       "7325                             1         0         0         1            0   \n",
       "21810                            0         1         0         1            0   \n",
       "4917                             1         0         0         0            0   \n",
       "639                              0         1         0         1            0   \n",
       "\n",
       "       major_eng  major_stat  \n",
       "587            0           0  \n",
       "3065           0           0  \n",
       "8435           0           0  \n",
       "3110           0           0  \n",
       "16372          0           0  \n",
       "...          ...         ...  \n",
       "16608          0           0  \n",
       "7325           0           0  \n",
       "21810          0           0  \n",
       "4917           0           1  \n",
       "639            0           0  \n",
       "\n",
       "[2110 rows x 18 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = pline.fit_transform(sur_X_train, sur_y_train)\n",
    "X_test = pline.transform(sur_X_test)\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "cef9476b-f69e-4dc2-a183-ca3592dee683",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "587      Software Engineer\n",
       "3065        Data Scientist\n",
       "8435        Data Scientist\n",
       "3110        Data Scientist\n",
       "16372    Software Engineer\n",
       "               ...        \n",
       "16608    Software Engineer\n",
       "7325     Software Engineer\n",
       "21810       Data Scientist\n",
       "4917        Data Scientist\n",
       "639         Data Scientist\n",
       "Name: Q6, Length: 2110, dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sur_y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44190ffb-9db1-4f25-8fc5-ce2abf61bec9",
   "metadata": {},
   "source": [
    "Time to create a module I will call in other notebooks as I move along"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "331ef79f-d575-4ed2-9a91-26c193d3b356",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%writefile EffectiveXGBoost.py\n",
    "# #\n",
    "# #\n",
    "# # (c) Ricky Macharm, MScFE\n",
    "# # https://SisengAI.com\n",
    "# #\n",
    "# #\n",
    "\n",
    "# import pandas as pd\n",
    "# import urllib.request\n",
    "# import zipfile\n",
    "\n",
    "# from feature_engine import encoding, imputation\n",
    "# from sklearn import base, pipeline\n",
    "\n",
    "# from sklearn import model_selection\n",
    "\n",
    "# def extract_dataset(src_path: str, dst_path: str, dataset: str) -> pd.DataFrame: \n",
    "#     \"\"\"Extracts a chosen dataset from a zip file and read it into a pandas\n",
    "#     DataFrame.\n",
    "    \n",
    "#     Parameters:\n",
    "#     ------------\n",
    "#     src_path: str\n",
    "#         URL or path of the zip file to be downloaded.\n",
    "#     dst_path: str\n",
    "#         Local file path where the zip file will be written.\n",
    "#     dataset: str\n",
    "#         Name of the particular file inside the zip file to be \n",
    "#         read into a DataFrame.\n",
    "    \n",
    "#     Returns:\n",
    "#     ---------\n",
    "#     pandas.core.frame.DataFrame: \n",
    "#         DataFrame containing the contents of the selected dataset.\n",
    "        \n",
    "#     \"\"\"\n",
    "    \n",
    "#     # using `urllib.request` module to open the URL specified in `src_path`.    \n",
    "#     open_path = urllib.request.urlopen(src_path)\n",
    "    \n",
    "#     # reads the contents of the file object `open_path`\n",
    "#     data = open_path.read()\n",
    "    \n",
    "#     # using context management to write the file in the destination path or folder\n",
    "#     with open(dst_path, mode='wb') as f:\n",
    "#         f.write(data)\n",
    "       \n",
    "#     # using context management to extract the saved zip file.\n",
    "#     with zipfile.ZipFile(dst_path) as z:\n",
    "#         all_df = pd.read_csv(z.open(dataset))\n",
    "#         df_ = all_df.iloc[0]\n",
    "#         df = all_df.iloc[1:]\n",
    "#         return df\n",
    "    \n",
    "    \n",
    "\n",
    "#     def topx(pser: pd.Series, x: int = 5, default='other') -> pd.Series:\n",
    "#         \"\"\"Replace all values in a Pandas Series that are not among\n",
    "#         the top `x` most frequent values with a default value.\n",
    "\n",
    "#         Parameters\n",
    "#         ----------\n",
    "#         pser : pd.Series\n",
    "#             The input pandas Series.\n",
    "#         x : int, optional\n",
    "#             The number of most frequent values to keep. The\n",
    "#             default value is 5.\n",
    "#         default : str, optional\n",
    "#             The default value to use for values that are not among\n",
    "#             the top `x` most frequent values. The default value is\n",
    "#             'other'.\n",
    "\n",
    "#         Returns:\n",
    "#         ---------\n",
    "#         pandas.Series: \n",
    "#             Series containing the results.\n",
    "#         \"\"\"\n",
    "\n",
    "#         val_count = pser.value_counts()\n",
    "#         return (pser.where(pser\n",
    "#                           .isin(val_count.index[:x]),\n",
    "#                           default)\n",
    "#                )\n",
    "\n",
    "\n",
    "# def prep_data(pdf: pd.DataFrame) -> pd.DataFrame:\n",
    "#     \"\"\" This function prepares the rurvey data and returns a new DataFrame.\n",
    "#     The prepatations include extracting and transforming certain\n",
    "#     columns, renaming some, and selecting a subset of columns.\n",
    "    \n",
    "#     Parameters\n",
    "#     ----------\n",
    "#     pdf : pd.DataFrame\n",
    "#         The input DataFrame containing the survey data.\n",
    "    \n",
    "#     Returns\n",
    "#     -------\n",
    "#     pd.DataFrame:\n",
    "#         The new DataFrame with the modified and selected columns.\n",
    "#     \"\"\"\n",
    "    \n",
    "#     return (pdf\n",
    "#             .assign(age=(pdf['Q2']\n",
    "#                          .str.slice(0,2)\n",
    "#                          .astype(int)\n",
    "#                         ),\n",
    "#                    education=(pdf\n",
    "#                              ['Q4']\n",
    "#                              .replace({'Doctoral degree' : 20, 'Bachelor’s degree' : 16, \n",
    "#                                        'Master’s degree' : 18,\n",
    "#                                    'Professional degree' : 19,\n",
    "#                                    'Some college/university study without earning a bachelor’s degree': 13,\n",
    "#                                    'I prefer not to answer' : None, \n",
    "#                                     'No formal education past high school': 12})\n",
    "#                             ),\n",
    "#                     major=(pdf['Q5']\n",
    "#                            .pipe(topx, x=3)\n",
    "#                            .replace({'Engineering (non-computer focused)': 'eng',\n",
    "#                                        'Computer science (software engineering, etc.)': 'cs',\n",
    "#                                        'Mathematics or statistics' : 'stat',})\n",
    "#                           ),\n",
    "#                     years_exp=(pdf\n",
    "#                              ['Q8']\n",
    "#                              .str.replace('+', '', regex=False)\n",
    "#                              .str.split('-', expand=True)\n",
    "#                              .iloc[:,0]\n",
    "#                              .astype(float)\n",
    "#                             ),\n",
    "#                     compensation=(pdf\n",
    "#                              ['Q9']\n",
    "#                              .str.replace('+', '', regex=False)\n",
    "#                              .str.replace(',','')\n",
    "#                              .replace({'500000':'500',\n",
    "#                                       'I do not wish to disclose my approximate yearly compensation':'0'})\n",
    "#                              .str.split('-', expand=True)\n",
    "#                              .iloc[:,0]\n",
    "#                              .fillna(0)\n",
    "#                              .astype(int)\n",
    "#                              .mul(1_000)\n",
    "#                             ),\n",
    "#                     python=(pdf\n",
    "#                              ['Q16_Part_1']\n",
    "#                              .fillna(0)\n",
    "#                              .replace({'Python': 1})\n",
    "#                             ),\n",
    "#                     r=(pdf\n",
    "#                          ['Q16_Part_2']\n",
    "#                          .fillna(0)\n",
    "#                          .replace({'R': 1})\n",
    "#                         ),\n",
    "#                     sql=(pdf\n",
    "#                          ['Q16_Part_3']\n",
    "#                          .fillna(0)\n",
    "#                          .replace({'SQL': 1})\n",
    "#                         )\n",
    "#                     )\n",
    "#             .rename(columns=lambda col:col.replace(' ', '_'))\n",
    "#             .loc[:, 'Q1,Q3,age,education,major,years_exp,compensation,python,r,sql'.split(',')]\n",
    "#     )\n",
    "\n",
    "\n",
    "# class PrepDataTransformer(base.BaseEstimator,\n",
    "#     base.TransformerMixin):\n",
    "#     \"\"\"\n",
    "#     This transformer takes a Pandas DataFrame containing our survey \n",
    "#     data as input and returns a new version of the DataFrame. \n",
    "    \n",
    "#     ----------\n",
    "#     ycol : str, optional\n",
    "#         The name of the column to be used as the target variable.\n",
    "#         If not specified, the target variable will not be set.\n",
    "#     Attributes\n",
    "#     ----------\n",
    "#     ycol : str\n",
    "#         The name of the column to be used as the target variable.\n",
    "#     \"\"\"\n",
    "#     def __init__(self, ycol=None):\n",
    "#         self.ycol = ycol\n",
    "    \n",
    "#     def transform(self, X):\n",
    "#         return prep_data(X)\n",
    "\n",
    "#     def fit(self, X, y=None):\n",
    "#         return self\n",
    "\n",
    "# def prepX_y(df, y_col):\n",
    "#     raw = (df\n",
    "#     .query('Q3.isin([\"United States of America\", \"China\", \"India\"]) '\n",
    "#     'and Q6.isin([\"Data Scientist\", \"Software Engineer\"])')\n",
    "#     )\n",
    "#     return raw.drop(columns=[y_col]), raw[y_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d25b819d-fd33-48c1-b27d-b00b30522981",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}