translation_transformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kprokofi/cv_text_classification/blob/main/translation_transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup"
      ],
      "metadata": {
        "id": "CLaeRh89Jt5w"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U spacy -q\n",
        "!python -m spacy download en_core_web_sm\n",
        "!python -m spacy download ru_core_news_sm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2brPTmsLk-35",
        "outputId": "ee965dfb-3ead-428b-c15f-0705ea2faa3e"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 6.0 MB 5.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 451 kB 48.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 10.1 MB 35.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 42 kB 1.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 628 kB 40.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 181 kB 51.2 MB/s \n",
            "\u001b[?25hCollecting en-core-web-sm==3.2.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0-py3-none-any.whl (13.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 13.9 MB 5.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.3.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-sm==3.2.0) (3.2.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (21.3)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.4.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.8.2)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.10.0.2)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (4.62.3)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.6)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.1)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.23.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (57.4.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.8.2)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.4.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.8)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (8.0.13)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.4.2)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.3.0)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.6.1)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.19.5)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.11.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (5.2.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.1)\n",
            "Installing collected packages: en-core-web-sm\n",
            "  Attempting uninstall: en-core-web-sm\n",
            "    Found existing installation: en-core-web-sm 2.2.5\n",
            "    Uninstalling en-core-web-sm-2.2.5:\n",
            "      Successfully uninstalled en-core-web-sm-2.2.5\n",
            "Successfully installed en-core-web-sm-3.2.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n",
            "Collecting ru-core-news-sm==3.2.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/ru_core_news_sm-3.2.0/ru_core_news_sm-3.2.0-py3-none-any.whl (16.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 16.4 MB 1.8 MB/s \n",
            "\u001b[?25hCollecting pymorphy2>=0.9\n",
            "  Downloading pymorphy2-0.9.1-py3-none-any.whl (55 kB)\n",
            "\u001b[K     |████████████████████████████████| 55 kB 2.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.3.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from ru-core-news-sm==3.2.0) (3.2.1)\n",
            "Collecting pymorphy2-dicts-ru<3.0,>=2.4\n",
            "  Downloading pymorphy2_dicts_ru-2.4.417127.4579844-py2.py3-none-any.whl (8.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 8.2 MB 8.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: docopt>=0.6 in /usr/local/lib/python3.7/dist-packages (from pymorphy2>=0.9->ru-core-news-sm==3.2.0) (0.6.2)\n",
            "Collecting dawg-python>=0.7.1\n",
            "  Downloading DAWG_Python-0.7.2-py2.py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (1.0.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2.11.3)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (0.8.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (21.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2.4.2)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (0.6.1)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (3.3.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (3.0.8)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (1.8.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2.23.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (1.0.6)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (8.0.13)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (4.62.3)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (0.4.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (57.4.0)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (3.10.0.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (0.4.1)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (1.19.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (5.2.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (3.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.3.0,>=3.2.0->ru-core-news-sm==3.2.0) (2.0.1)\n",
            "Installing collected packages: pymorphy2-dicts-ru, dawg-python, pymorphy2, ru-core-news-sm\n",
            "Successfully installed dawg-python-0.7.2 pymorphy2-0.9.1 pymorphy2-dicts-ru-2.4.417127.4579844 ru-core-news-sm-3.2.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('ru_core_news_sm')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "%matplotlib inline"
      ],
      "metadata": {
        "id": "Qdg6OdTte_O1"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Full dataset loading"
      ],
      "metadata": {
        "id": "3Em0Fn6bJgrY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "ParaCrawl Rus-Eng dataset was choosen for experiments, link:\n",
        "https://lindat.mff.cuni.cz/repository/xmlui/handle/11372/LRT-2610"
      ],
      "metadata": {
        "id": "kGQEzs-fJ9CO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget \"https://lindat.mff.cuni.cz/repository/xmlui/bitstream/handle/11372/LRT-2610/paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz?sequence=9&isAllowed=y\" -O out.tgz  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KzX8x3kgfboV",
        "outputId": "5a10138d-07e5-49c8-fd6e-af6d75d7cc59"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-12-15 11:14:19--  https://lindat.mff.cuni.cz/repository/xmlui/bitstream/handle/11372/LRT-2610/paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz?sequence=9&isAllowed=y\n",
            "Resolving lindat.mff.cuni.cz (lindat.mff.cuni.cz)... 195.113.20.140\n",
            "Connecting to lindat.mff.cuni.cz (lindat.mff.cuni.cz)|195.113.20.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 667981874 (637M) [application/x-gzip]\n",
            "Saving to: ‘out.tgz’\n",
            "\n",
            "out.tgz             100%[===================>] 637.04M  25.1MB/s    in 27s     \n",
            "\n",
            "2021-12-15 11:14:48 (23.6 MB/s) - ‘out.tgz’ saved [667981874/667981874]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!tar -xvf out.tgz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_VqZWF2XnqtL",
        "outputId": "c31aaca5-f02a-48f0-f080-2dc5e39a20b4"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "paracrawl-release1.en-ru.zipporah0-dedup-clean.en\n",
            "paracrawl-release1.en-ru.zipporah0-dedup-clean.ru\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lines_eng = open('paracrawl-release1.en-ru.zipporah0-dedup-clean.en', 'r').readlines()\n",
        "lines_rus = open('paracrawl-release1.en-ru.zipporah0-dedup-clean.ru', 'r').readlines()"
      ],
      "metadata": {
        "id": "nA3n2MWYgK7r"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lines_rus[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JRAe_zBGgUo5",
        "outputId": "e88c4ffb-2cd0-4d68-ef39-a86e73918f39"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Email: info@e-e-e.cz\\n',\n",
              " 'Фирма зарегистрирована в Городском суде в г. Праге, раздел Б, вкладыш 14857.\\n',\n",
              " 'Новости:\\n',\n",
              " '22.02.09\\n',\n",
              " 'С января месяца 2009 г. открыты новые офисные помещения на улице V Kolkovně 3 (В Колковне, д. 3), Прага-1. Данные помещения находятся поблизости всех банков-партнеров.\\n',\n",
              " 'O НАС Новости Референции Контакт\\n',\n",
              " 'Новости\\n',\n",
              " '02 / 2009\\n',\n",
              " 'Открыты новые офисные помещения (22. 02.)\\n',\n",
              " 'Компания АО «East Export Engineering» («EEE») исходит из успехов чешских фирм и банков в области экспортного финансирования; она поставила перед собой цель далее развивать поддержку финансирования чешских и зарубежных проектов с участием чешских фирм в качестве подрядчиков/поставщиков.\\n']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lines_eng[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mA7ctlJWgjyv",
        "outputId": "a82b4349-c648-4ebf-cae6-c0c91da3eda4"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['E-mail: info@e-e-e.cz\\n',\n",
              " 'The company has been registered with the Municipal Court in Prague, in Section B, File 14857.\\n',\n",
              " 'News:\\n',\n",
              " '22.02.09\\n',\n",
              " 'Since January 2009 new office premises in the street V Kolkovně 3, Prague 1, have been opened. These premises are situated in a close vicinity of all our partner banks.\\n',\n",
              " 'About us News References Contacts\\n',\n",
              " 'News\\n',\n",
              " '02 / 2009\\n',\n",
              " 'New projects (22. 02.) New office premises have been opened (19. 02.)\\n',\n",
              " 'East Export Engineering a.s. (EEE) has reassumed the previous success of Czech companies and banks in the field of export financing and commenced its activity with the aim to further develop the support of financing of home and foreign schemes involving supplier participation of Czech companies.\\n']"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(lines_eng)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JwrNFAO7hXSC",
        "outputId": "2a878af1-9a73-49da-ab52-c21fe032338d"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "12061155"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# EDA"
      ],
      "metadata": {
        "id": "mx2bQf19kRki"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check elements length"
      ],
      "metadata": {
        "id": "I0ytGWtKeL1V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_statistics(lines_rus, lines_eng):\n",
        "    lenghts = {}\n",
        "    for data, key in [(lines_rus, 'rus'), (lines_eng, 'eng')]:\n",
        "        lenghts[key] = np.array([len(sent) for sent in data], dtype=int)\n",
        "\n",
        "    def collect_statistics(arr):\n",
        "        res = {}\n",
        "        res['mean'] = arr.mean()\n",
        "        res['argmin'] = np.argmin(arr)\n",
        "        res['argmax'] = np.argmax(arr)\n",
        "        res['min'] = arr.min()\n",
        "        res['max'] = arr.max()\n",
        "        return res\n",
        "\n",
        "    stat_rus = collect_statistics(lenghts['rus'])\n",
        "    stat_eng = collect_statistics(lenghts['eng'])\n",
        "    ratious = lenghts['rus'] / lenghts['eng']\n",
        "    stat_ratious = collect_statistics(ratious)\n",
        "\n",
        "    def get_pair(idx):\n",
        "        return lines_eng[idx], lines_rus[idx]\n",
        "\n",
        "    smallest = get_pair(stat_eng['argmin'])\n",
        "    longest = get_pair(stat_eng['argmax'])\n",
        "    smallest_ratio = get_pair(stat_ratious['argmin'])\n",
        "    biggest_ratio = get_pair(stat_ratious['argmax'])\n",
        "\n",
        "    len_stat_srt = f'Lenght statistics:\\nRus:{stat_rus}\\nEng{stat_eng}\\nRus / Eng {stat_ratious}'\n",
        "    extreems_ex_str = f'Pairs\\nSmallest: {smallest}\\n'\\\n",
        "                      f'Longest: {longest}\\n'\\\n",
        "                      f'Smallest ratio: {smallest_ratio}\\n'\\\n",
        "                      f'Biggest ratio: {biggest_ratio}'\n",
        "                    \n",
        "    counts = f'Counts of pairs: {len(lines_rus)}'\n",
        "    descr = len_stat_srt + '\\n' + extreems_ex_str + '\\n' + counts\n",
        "    retval = {}\n",
        "    retval['rus'] = stat_rus\n",
        "    retval['eng'] = stat_eng\n",
        "    retval['ratious'] = ratious\n",
        "    retval['len'] = lenghts\n",
        "\n",
        "    return descr, retval\n",
        "\n",
        "stats_init = get_statistics(lines_rus, lines_eng) \n",
        "print(stats_init[0])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jw_g_48CevQI",
        "outputId": "166eeba3-07dc-4789-c231-578386a372df"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Lenght statistics:\n",
            "Rus:{'mean': 79.00969981730606, 'argmin': 677, 'argmax': 9573105, 'min': 2, 'max': 15554}\n",
            "Eng{'mean': 82.27549426236543, 'argmin': 677, 'argmax': 9573105, 'min': 2, 'max': 15554}\n",
            "Rus / Eng {'mean': 1.0397173034121685, 'argmin': 5459424, 'argmax': 7310074, 'min': 0.015503875968992248, 'max': 56.0}\n",
            "Pairs\n",
            "Smallest: ('?\\n', '?\\n')\n",
            "Longest: ('\\'))}},_connectDatepicker:function(b,f){var j=a(b);f.append=a([]);f.trigger=a([]);if(!j.hasClass(this.markerClassName)){this._attachments(j,f);j.addClass(this.markerClassName).keydown(this._doKeyDown).keypress(this._doKeyPress).keyup(this._doKeyUp).bind(\"setData.datepicker\", function(l,o,n){f.settings[o]=n}).bind(\"getData.datepicker\",function(l,o){return this._get(f,o)});this._autoSize(f);a.data(b,\"datepicker\",f);f.settings.disabled&&this._disableDatepicker(b)}},_attachments:function(b,f){var j=this._get(f,\"appendText\"),l=this._get(f,\"isRTL\");f.append&&f.append.remove();if(j){f.append=a(\\' \\'+j+\" \");b[l?\"before\":\"after\"](f.append)}b.unbind(\"focus\",this._showDatepicker);f.trigger&&f.trigger.remove();j=this._get(f,\"showOn\");if(j== \"focus\"j==\"both\")b.focus(this._showDatepicker);if(j==\"button\"j==\"both\"){j=this._get(f,\"buttonText\");var o=this._get(f,\"buttonImage\");f.trigger=a(this._get(f,\"buttonImageOnly\")?a(\"\").addClass(this._triggerClass).attr({src:o,alt:j,title:j}):a(\\'\\').addClass(this._triggerClass).html(o==\"\"?j:a(\"\").attr({src:o,alt:j,title:j})));b[l?\"before\":\"after\"](f.trigger);f.trigger.click(function(){a.datepicker._datepickerShowing&&a.datepicker._lastInput==b[0]?a.datepicker._hideDatepicker(): a.datepicker._showDatepicker(b[0]);return false})}},_autoSize:function(b){if(this._get(b,\"autoSize\")&&!b.inline){var f=new Date(2009,11,20),j=this._get(b,\"dateFormat\");if(j.match(/[DM]/)){var l=function(o){for(var n=0,k=0,m=0;mn){n=o[m].length;k=m}return k};f.setMonth(l(this._get(b,j.match(/MM/)?\"monthNames\":\"monthNamesShort\")));f.setDate(l(this._get(b,j.match(/DD/)?\"dayNames\":\"dayNamesShort\"))+20-f.getDay())}b.input.attr(\"size\",this._formatDate(b,f).length)}},_inlineDatepicker:function(b, f){var j=a(b);if(!j.hasClass(this.markerClassName)){j.addClass(this.markerClassName).append(f.dpDiv).bind(\"setData.datepicker\",function(l,o,n){f.settings[o]=n}).bind(\"getData.datepicker\",function(l,o){return this._get(f,o)});a.data(b,\"datepicker\",f);this._setDate(f,this._getDefaultDate(f),true);this._updateDatepicker(f);this._updateAlternate(f);f.settings.disabled&&this._disableDatepicker(b);f.dpDiv.css(\"display\",\"block\")}},_dialogDatepicker:function(b,f,j,l,o){b=this._dialogInst;if(!b){this.uuid+= 1;this._dialogInput=a(\\'\\');this._dialogInput.keydown(this._doKeyDown);a(\"body\").append(this._dialogInput);b=this._dialogInst=this._newInst(this._dialogInput,false);b.settings={};a.data(this._dialogInput[0],\"datepicker\",b)}h(b.settings,l{});f=f&&f.constructor==Date?this._formatDate(b,f):f;this._dialogInput.val(f);this._pos=o?o.length?o:[o.pageX,o.pageY]:null;if(!this._pos)this._pos=[document.documentElement.clientWidth/ 2-100+(document.documentElement.scrollLeftdocument.body.scrollLeft),document.documentElement.clientHeight/2-150+(document.documentElement.scrollTopdocument.body.scrollTop)];this._dialogInput.css(\"left\",this._pos[0]+20+\"px\").css(\"top\",this._pos[1]+\"px\");b.settings.onSelect=j;this._inDialog=true;this.dpDiv.addClass(this._dialogClass);this._showDatepicker(this._dialogInput[0]);a.blockUI&&a.blockUI(this.dpDiv);a.data(this._dialogInput[0],\"datepicker\",b);return this},_destroyDatepicker:function(b){var f= a(b),j=a.data(b,\"datepicker\");if(f.hasClass(this.markerClassName)){var l=b.nodeName.toLowerCase();a.removeData(b,\"datepicker\");if(l==\"input\"){j.append.remove();j.trigger.remove();f.removeClass(this.markerClassName).unbind(\"focus\",this._showDatepicker).unbind(\"keydown\",this._doKeyDown).unbind(\"keypress\",this._doKeyPress).unbind(\"keyup\",this._doKeyUp)}else if(l==\"div\"l==\"span\")f.removeClass(this.markerClassName).empty()}},_enableDatepicker:function(b){var f=a(b),j=a.data(b,\"datepicker\");if(f.hasClass(this.markerClassName)){var l= b.nodeName.toLowerCase();if(l==\"input\"){b.disabled=false;j.trigger.filter(\"button\").each(function(){this.disabled=false}).end().filter(\"img\").css({opacity:\"1.0\",cursor:\"\"})}else if(l==\"div\"l==\"span\"){f=f.children(\".\"+this._inlineClass);f.children().removeClass(\"ui-state-disabled\");f.find(\"select.ui-datepicker-month, select.ui-datepicker-year\").removeAttr(\"disabled\")}this._disabledInputs=a.map(this._disabledInputs,function(o){return o==b?null:o})}},_disableDatepicker:function(b){var f=a(b),j=a.data(b, \"datepicker\");if(f.hasClass(this.markerClassName)){var l=b.nodeName.toLowerCase();if(l==\"input\"){b.disabled=true;j.trigger.filter(\"button\").each(function(){this.disabled=true}).end().filter(\"img\").css({opacity:\"0.5\",cursor:\"default\"})}else if(l==\"div\"l==\"span\"){f=f.children(\".\"+this._inlineClass);f.children().addClass(\"ui-state-disabled\");f.find(\"select.ui-datepicker-month, select.ui-datepicker-year\").attr(\"disabled\",\"disabled\")}this._disabledInputs=a.map(this._disabledInputs,function(o){return o== b?null:o});this._disabledInputs[this._disabledInputs.length]=b}},_isDisabledDatepicker:function(b){if(!b)return false;for(var f=0;f-1}},_doKeyUp:function(b){b=a.datepicker._getInst(b.target);if(b.input.val()!=b.lastVal)try{if(a.datepicker.parseDate(a.datepicker._get(b,\"dateFormat\"),b.input?b.input.val():null,a.datepicker._getFormatConfig(b))){a.datepicker._setDateFromField(b);a.datepicker._updateAlternate(b);a.datepicker._updateDatepicker(b)}}catch(f){a.datepicker.log(f)}return true},_showDatepicker:function(b){b=b.targetb;if(b.nodeName.toLowerCase()!=\"input\")b=a(\"input\", b.parentNode)[0];if(!(a.datepicker._isDisabledDatepicker(b)a.datepicker._lastInput==b)){var f=a.datepicker._getInst(b);if(a.datepicker._curInst&&a.datepicker._curInst!=f){a.datepicker._datepickerShowing&&a.datepicker._triggerOnClose(a.datepicker._curInst);a.datepicker._curInst.dpDiv.stop(true,true)}var j=a.datepicker._get(f,\"beforeShow\");j=j?j.apply(b,[b,f]):{};if(j!==false){h(f.settings,j);f.lastVal=null;a.datepicker._lastInput=b;a.datepicker._setDateFromField(f);if(a.datepicker._inDialog)b.value= \"\";if(!a.datepicker._pos){a.datepicker._pos=a.datepicker._findPos(b);a.datepicker._pos[1]+=b.offsetHeight}var l=false;a(b).parents().each(function(){l=a(this).css(\"position\")==\"fixed\";return!l});if(l&&a.browser.opera){a.datepicker._pos[0]-=document.documentElement.scrollLeft;a.datepicker._pos[1]-=document.documentElement.scrollTop}j={left:a.datepicker._pos[0],top:a.datepicker._pos[1]};a.datepicker._pos=null;f.dpDiv.empty();f.dpDiv.css({position:\"absolute\",display:\"block\",top:\"-1000px\"});a.datepicker._updateDatepicker(f); j=a.datepicker._checkOffset(f,j,l);f.dpDiv.css({position:a.datepicker._inDialog&&a.blockUI?\"static\":l?\"fixed\":\"absolute\",display:\"none\",left:j.left+\"px\",top:j.top+\"px\"});if(!f.inline){j=a.datepicker._get(f,\"showAnim\");var o=a.datepicker._get(f,\"duration\"),n=function(){var k=f.dpDiv.find(\"iframe.ui-datepicker-cover\");if(k.length){var m=a.datepicker._getBorders(f.dpDiv);k.css({left:-m[0],top:-m[1],width:f.dpDiv.outerWidth(),height:f.dpDiv.outerHeight()})}};f.dpDiv.zIndex(a(b).zIndex()+1);a.datepicker._datepickerShowing= true;a.effects&&a.effects[j]?f.dpDiv.show(j,a.datepicker._get(f,\"showOptions\"),o,n):f.dpDiv[j\"show\"](j?o:null,n);if(!j!o)n();f.input.is(\":visible\")&&!f.input.is(\":disabled\")&&f.input.focus();a.datepicker._curInst=f}}}},_updateDatepicker:function(b){this.maxRows=4;var f=a.datepicker._getBorders(b.dpDiv);i=b;b.dpDiv.empty().append(this._generateHTML(b));var j=b.dpDiv.find(\"iframe.ui-datepicker-cover\");j.length&&j.css({left:-f[0],top:-f[1],width:b.dpDiv.outerWidth(),height:b.dpDiv.outerHeight()}); b.dpDiv.find(\".\"+this._dayOverClass+\" a\").mouseover();f=this._getNumberOfMonths(b);j=f[1];b.dpDiv.removeClass(\"ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4\").width(\"\");j>1&&b.dpDiv.addClass(\"ui-datepicker-multi-\"+j).css(\"width\",17*j+\"em\");b.dpDiv[(f[0]!=1f[1]!=1?\"add\":\"remove\")+\"Class\"](\"ui-datepicker-multi\");b.dpDiv[(this._get(b,\"isRTL\")?\"add\":\"remove\")+\"Class\"](\"ui-datepicker-rtl\");b==a.datepicker._curInst&&a.datepicker._datepickerShowing&&b.input&&b.input.is(\":visible\")&& !b.input.is(\":disabled\")&&b.input[0]!=document.activeElement&&b.input.focus();if(b.yearshtml){var l=b.yearshtml;setTimeout(function(){l===b.yearshtml&&b.yearshtml&&b.dpDiv.find(\"select.ui-datepicker-year:first\").replaceWith(b.yearshtml);l=b.yearshtml=null},0)}},_getBorders:function(b){var f=function(j){return{thin:1,medium:2,thick:3}[j]j};return[parseFloat(f(b.css(\"border-left-width\"))),parseFloat(f(b.css(\"border-top-width\")))]},_checkOffset:function(b,f,j){var l=b.dpDiv.outerWidth(),o=b.dpDiv.outerHeight(), n=b.input?b.input.outerWidth():0,k=b.input?b.input.outerHeight():0,m=document.documentElement.clientWidth+a(document).scrollLeft(),p=document.documentElement.clientHeight+a(document).scrollTop();f.left-=this._get(b,\"isRTL\")?l-n:0;f.left-=j&&f.left==b.input.offset().left?a(document).scrollLeft():0;f.top-=j&&f.top==b.input.offset().top+k?a(document).scrollTop():0;f.left-=Math.min(f.left,f.left+l>m&&m>l?Math.abs(f.left+l-m):0);f.top-=Math.min(f.top,f.top+o>p&&p>o?Math.abs(o+k):0);return f},_findPos:function(b){for(var f= this._get(this._getInst(b),\"isRTL\");b&&(b.type==\"hidden\"b.nodeType!=1a.expr.filters.hidden(b));)b=b[f?\"previousSibling\":\"nextSibling\"];b=a(b).offset();return[b.left,b.top]},_triggerOnClose:function(b){var f=this._get(b,\"onClose\");if(f)f.apply(b.input?b.input[0]:null,[b.input?b.input.val():\"\",b])},_hideDatepicker:function(b){var f=this._curInst;if(!(!fb&&f!=a.data(b,\"datepicker\")))if(this._datepickerShowing){b=this._get(f,\"showAnim\");var j=this._get(f,\"duration\"),l=function(){a.datepicker._tidyDialog(f); this._curInst=null};a.effects&&a.effects[b]?f.dpDiv.hide(b,a.datepicker._get(f,\"showOptions\"),j,l):f.dpDiv[b==\"slideDown\"?\"slideUp\":b==\"fadeIn\"?\"fadeOut\":\"hide\"](b?j:null,l);bl();a.datepicker._triggerOnClose(f);this._datepickerShowing=false;this._lastInput=null;if(this._inDialog){this._dialogInput.css({position:\"absolute\",left:\"0\",top:\"-100px\"});if(a.blockUI){a.unblockUI();a(\"body\").append(this.dpDiv)}}this._inDialog=false}},_tidyDialog:function(b){b.dpDiv.removeClass(this._dialogClass).unbind(\".ui-datepicker-calendar\")}, _checkExternalClick:function(b){if(a.datepicker._curInst){b=a(b.target);b[0].id!=a.datepicker._mainDivId&&b.parents(\"#\"+a.datepicker._mainDivId).length==0&&!b.hasClass(a.datepicker.markerClassName)&&!b.hasClass(a.datepicker._triggerClass)&&a.datepicker._datepickerShowing&&!(a.datepicker._inDialog&&a.blockUI)&&a.datepicker._hideDatepicker()}},_adjustDate:function(b,f,j){b=a(b);var l=this._getInst(b[0]);if(!this._isDisabledDatepicker(b[0])){this._adjustInstDate(l,f+(j==\"M\"?this._get(l,\"showCurrentAtPos\"): 0),j);this._updateDatepicker(l)}},_gotoToday:function(b){b=a(b);var f=this._getInst(b[0]);if(this._get(f,\"gotoCurrent\")&&f.currentDay){f.selectedDay=f.currentDay;f.drawMonth=f.selectedMonth=f.currentMonth;f.drawYear=f.selectedYear=f.currentYear}else{var j=new Date;f.selectedDay=j.getDate();f.drawMonth=f.selectedMonth=j.getMonth();f.drawYear=f.selectedYear=j.getFullYear()}this._notifyChange(f);this._adjustDate(b)},_selectMonthYear:function(b,f,j){b=a(b);var l=this._getInst(b[0]);l[\"selected\"+(j==\"M\"? \"Month\":\"Year\")]=l[\"draw\"+(j==\"M\"?\"Month\":\"Year\")]=parseInt(f.options[f.selectedIndex].value,10);this._notifyChange(l);this._adjustDate(b)},_selectDay:function(b,f,j,l){var o=a(b);if(!(a(l).hasClass(this._unselectableClass)this._isDisabledDatepicker(o[0]))){o=this._getInst(o[0]);o.selectedDay=o.currentDay=a(\"a\",l).html();o.selectedMonth=o.currentMonth=f;o.selectedYear=o.currentYear=j;this._selectDate(b,this._formatDate(o,o.currentDay,o.currentMonth,o.currentYear))}},_clearDate:function(b){b=a(b); this._getInst(b[0]);this._selectDate(b,\"\")},_selectDate:function(b,f){b=this._getInst(a(b)[0]);f=f!=null?f:this._formatDate(b);b.input&&b.input.val(f);this._updateAlternate(b);var j=this._get(b,\"onSelect\");if(j)j.apply(b.input?b.input[0]:null,[f,b]);else b.input&&b.input.trigger(\"change\");if(b.inline)this._updateDatepicker(b);else{this._hideDatepicker();this._lastInput=b.input[0];typeof b.input[0]!=\"object\"&&b.input.focus();this._lastInput=null}},_updateAlternate:function(b){var f=this._get(b,\"altField\"); if(f){var j=this._get(b,\"altFormat\")this._get(b,\"dateFormat\"),l=this._getDate(b),o=this.formatDate(j,l,this._getFormatConfig(b));a(f).each(function(){a(this).val(o)})}},noWeekends:function(b){b=b.getDay();return[b>0&&b<6,\"\"]},iso8601Week:function(b){b=new Date(b.getTime());b.setDate(b.getDate()+4-(b.getDay()7));var f=b.getTime();b.setMonth(0);b.setDate(1);return Math.floor(Math.round((f-b)/864E5)/7)+1},parseDate:function(b,f,j){if(b==nullf==null)throw\"Invalid arguments\";f=typeof f==\"object\"? f.toString():f+\"\";if(f==\"\")return null;var l=(j?j.shortYearCutoff:null)this._defaults.shortYearCutoff;l=typeof l!=\"string\"?l:(new Date).getFullYear()%100+parseInt(l,10);for(var o=(j?j.dayNamesShort:null)this._defaults.dayNamesShort,n=(j?j.dayNames:null)this._defaults.dayNames,k=(j?j.monthNamesShort:null)this._defaults.monthNamesShort,m=(j?j.monthNames:null)this._defaults.monthNames,p=j=-1,q=-1,s=-1,r=false,u=function(z){(z=H+1-1){p=1;q=s;do{l=this._getDaysInMonth(j,p-1);if(q<=l)break;p++;q-=l}while(1)}C=this._daylightSavingAdjust(new Date(j,p-1,q));if(C.getFullYear()!=jC.getMonth()+1!=pC.getDate()!=q)throw\"Invalid date\";return C},ATOM:\"yy-mm-dd\", COOKIE:\"D, dd M yy\",ISO_8601:\"yy-mm-dd\",RFC_822:\"D, d M y\",RFC_850:\"DD, dd-M-y\",RFC_1036:\"D, d M y\",RFC_1123:\"D, d M yy\",RFC_2822:\"D, d M yy\",RSS:\"D, d M y\",TICKS:\"!\",TIMESTAMP:\"@\",W3C:\"yy-mm-dd\",_ticksTo1970:(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*24*60*60*1E7,formatDate:function(b,f,j){if(!f)return\"\";var l=(j?j.dayNamesShort:null)this._defaults.dayNamesShort,o=(j?j.dayNames:null)this._defaults.dayNames,n=(j?j.monthNamesShort:null)this._defaults.monthNamesShort;j=(j?j.monthNames: null)this._defaults.monthNames;var k=function(u){(u=r+1 12?b.getHours()+2:0);return b},_setDate:function(b,f,j){var l=!f,o=b.selectedMonth,n=b.selectedYear;f=this._restrictMinMax(b,this._determineDate(b,f,new Date));b.selectedDay=b.currentDay=f.getDate();b.drawMonth=b.selectedMonth=b.currentMonth=f.getMonth();b.drawYear=b.selectedYear=b.currentYear=f.getFullYear();if((o!=b.selectedMonthn!=b.selectedYear)&&!j)this._notifyChange(b);this._adjustInstDate(b);if(b.input)b.input.val(l?\"\":this._formatDate(b))},_getDate:function(b){return!b.currentYearb.input&& b.input.val()==\"\"?null:this._daylightSavingAdjust(new Date(b.currentYear,b.currentMonth,b.currentDay))},_generateHTML:function(b){var f=new Date;f=this._daylightSavingAdjust(new Date(f.getFullYear(),f.getMonth(),f.getDate()));var j=this._get(b,\"isRTL\"),l=this._get(b,\"showButtonPanel\"),o=this._get(b,\"hideIfNoPrevNext\"),n=this._get(b,\"navigationAsDateFormat\"),k=this._getNumberOfMonths(b),m=this._get(b,\"showCurrentAtPos\"),p=this._get(b,\"stepMonths\"),q=k[0]!=1k[1]!=1,s=this._daylightSavingAdjust(!b.currentDay? new Date(9999,9,9):new Date(b.currentYear,b.currentMonth,b.currentDay)),r=this._getMinMaxDate(b,\"min\"),u=this._getMinMaxDate(b,\"max\");m=b.drawMonth-m;var v=b.drawYear;if(m<0){m+=12;v--}if(u){var w=this._daylightSavingAdjust(new Date(u.getFullYear(),u.getMonth()-k[0]*k[1]+1,u.getDate()));for(w=r&&ww;){m--;if(m<0){m=11;v--}}}b.drawMonth=m;b.drawYear=v;w=this._get(b,\"prevText\");w=!n?w:this.formatDate(w,this._daylightSavingAdjust(new Date(v,m-p,1)),this._getFormatConfig(b)); w=this._canAdjustMonth(b,-1,v,m)?\\' \\'+w+\" \":o?\"\":\\' \\'+w+\" \";var x=this._get(b,\"nextText\");x=!n?x:this.formatDate(x,this._daylightSavingAdjust(new Date(v, m+p,1)),this._getFormatConfig(b));o=this._canAdjustMonth(b,+1,v,m)?\\' \\'+x+\" \":o?\"\":\\' \\'+x+\" \";p=this._get(b,\"currentText\");x=this._get(b,\"gotoCurrent\")&& b.currentDay?s:f;p=!n?p:this.formatDate(p,x,this._getFormatConfig(b));n=!b.inline?\\'\\'+this._get(b,\"closeText\")+\"\":\"\";l=l?\\'\\n', '\\'))}},_connectDatepicker:function(b,f){var j=a(b);f.append=a([]);f.trigger=a([]);if(!j.hasClass(this.markerClassName)){this._attachments(j,f);j.addClass(this.markerClassName).keydown(this._doKeyDown).keypress(this._doKeyPress).keyup(this._doKeyUp).bind(\"setData.datepicker\", function(l,o,n){f.settings[o]=n}).bind(\"getData.datepicker\",function(l,o){return this._get(f,o)});this._autoSize(f);a.data(b,\"datepicker\",f);f.settings.disabled&&this._disableDatepicker(b)}},_attachments:function(b,f){var j=this._get(f,\"appendText\"),l=this._get(f,\"isRTL\");f.append&&f.append.remove();if(j){f.append=a(\\' \\'+j+\" \");b[l?\"before\":\"after\"](f.append)}b.unbind(\"focus\",this._showDatepicker);f.trigger&&f.trigger.remove();j=this._get(f,\"showOn\");if(j== \"focus\"j==\"both\")b.focus(this._showDatepicker);if(j==\"button\"j==\"both\"){j=this._get(f,\"buttonText\");var o=this._get(f,\"buttonImage\");f.trigger=a(this._get(f,\"buttonImageOnly\")?a(\"\").addClass(this._triggerClass).attr({src:o,alt:j,title:j}):a(\\'\\').addClass(this._triggerClass).html(o==\"\"?j:a(\"\").attr({src:o,alt:j,title:j})));b[l?\"before\":\"after\"](f.trigger);f.trigger.click(function(){a.datepicker._datepickerShowing&&a.datepicker._lastInput==b[0]?a.datepicker._hideDatepicker(): a.datepicker._showDatepicker(b[0]);return false})}},_autoSize:function(b){if(this._get(b,\"autoSize\")&&!b.inline){var f=new Date(2009,11,20),j=this._get(b,\"dateFormat\");if(j.match(/[DM]/)){var l=function(o){for(var n=0,k=0,m=0;mn){n=o[m].length;k=m}return k};f.setMonth(l(this._get(b,j.match(/MM/)?\"monthNames\":\"monthNamesShort\")));f.setDate(l(this._get(b,j.match(/DD/)?\"dayNames\":\"dayNamesShort\"))+20-f.getDay())}b.input.attr(\"size\",this._formatDate(b,f).length)}},_inlineDatepicker:function(b, f){var j=a(b);if(!j.hasClass(this.markerClassName)){j.addClass(this.markerClassName).append(f.dpDiv).bind(\"setData.datepicker\",function(l,o,n){f.settings[o]=n}).bind(\"getData.datepicker\",function(l,o){return this._get(f,o)});a.data(b,\"datepicker\",f);this._setDate(f,this._getDefaultDate(f),true);this._updateDatepicker(f);this._updateAlternate(f);f.settings.disabled&&this._disableDatepicker(b);f.dpDiv.css(\"display\",\"block\")}},_dialogDatepicker:function(b,f,j,l,o){b=this._dialogInst;if(!b){this.uuid+= 1;this._dialogInput=a(\\'\\');this._dialogInput.keydown(this._doKeyDown);a(\"body\").append(this._dialogInput);b=this._dialogInst=this._newInst(this._dialogInput,false);b.settings={};a.data(this._dialogInput[0],\"datepicker\",b)}h(b.settings,l{});f=f&&f.constructor==Date?this._formatDate(b,f):f;this._dialogInput.val(f);this._pos=o?o.length?o:[o.pageX,o.pageY]:null;if(!this._pos)this._pos=[document.documentElement.clientWidth/ 2-100+(document.documentElement.scrollLeftdocument.body.scrollLeft),document.documentElement.clientHeight/2-150+(document.documentElement.scrollTopdocument.body.scrollTop)];this._dialogInput.css(\"left\",this._pos[0]+20+\"px\").css(\"top\",this._pos[1]+\"px\");b.settings.onSelect=j;this._inDialog=true;this.dpDiv.addClass(this._dialogClass);this._showDatepicker(this._dialogInput[0]);a.blockUI&&a.blockUI(this.dpDiv);a.data(this._dialogInput[0],\"datepicker\",b);return this},_destroyDatepicker:function(b){var f= a(b),j=a.data(b,\"datepicker\");if(f.hasClass(this.markerClassName)){var l=b.nodeName.toLowerCase();a.removeData(b,\"datepicker\");if(l==\"input\"){j.append.remove();j.trigger.remove();f.removeClass(this.markerClassName).unbind(\"focus\",this._showDatepicker).unbind(\"keydown\",this._doKeyDown).unbind(\"keypress\",this._doKeyPress).unbind(\"keyup\",this._doKeyUp)}else if(l==\"div\"l==\"span\")f.removeClass(this.markerClassName).empty()}},_enableDatepicker:function(b){var f=a(b),j=a.data(b,\"datepicker\");if(f.hasClass(this.markerClassName)){var l= b.nodeName.toLowerCase();if(l==\"input\"){b.disabled=false;j.trigger.filter(\"button\").each(function(){this.disabled=false}).end().filter(\"img\").css({opacity:\"1.0\",cursor:\"\"})}else if(l==\"div\"l==\"span\"){f=f.children(\".\"+this._inlineClass);f.children().removeClass(\"ui-state-disabled\");f.find(\"select.ui-datepicker-month, select.ui-datepicker-year\").removeAttr(\"disabled\")}this._disabledInputs=a.map(this._disabledInputs,function(o){return o==b?null:o})}},_disableDatepicker:function(b){var f=a(b),j=a.data(b, \"datepicker\");if(f.hasClass(this.markerClassName)){var l=b.nodeName.toLowerCase();if(l==\"input\"){b.disabled=true;j.trigger.filter(\"button\").each(function(){this.disabled=true}).end().filter(\"img\").css({opacity:\"0.5\",cursor:\"default\"})}else if(l==\"div\"l==\"span\"){f=f.children(\".\"+this._inlineClass);f.children().addClass(\"ui-state-disabled\");f.find(\"select.ui-datepicker-month, select.ui-datepicker-year\").attr(\"disabled\",\"disabled\")}this._disabledInputs=a.map(this._disabledInputs,function(o){return o== b?null:o});this._disabledInputs[this._disabledInputs.length]=b}},_isDisabledDatepicker:function(b){if(!b)return false;for(var f=0;f-1}},_doKeyUp:function(b){b=a.datepicker._getInst(b.target);if(b.input.val()!=b.lastVal)try{if(a.datepicker.parseDate(a.datepicker._get(b,\"dateFormat\"),b.input?b.input.val():null,a.datepicker._getFormatConfig(b))){a.datepicker._setDateFromField(b);a.datepicker._updateAlternate(b);a.datepicker._updateDatepicker(b)}}catch(f){a.datepicker.log(f)}return true},_showDatepicker:function(b){b=b.targetb;if(b.nodeName.toLowerCase()!=\"input\")b=a(\"input\", b.parentNode)[0];if(!(a.datepicker._isDisabledDatepicker(b)a.datepicker._lastInput==b)){var f=a.datepicker._getInst(b);if(a.datepicker._curInst&&a.datepicker._curInst!=f){a.datepicker._datepickerShowing&&a.datepicker._triggerOnClose(a.datepicker._curInst);a.datepicker._curInst.dpDiv.stop(true,true)}var j=a.datepicker._get(f,\"beforeShow\");j=j?j.apply(b,[b,f]):{};if(j!==false){h(f.settings,j);f.lastVal=null;a.datepicker._lastInput=b;a.datepicker._setDateFromField(f);if(a.datepicker._inDialog)b.value= \"\";if(!a.datepicker._pos){a.datepicker._pos=a.datepicker._findPos(b);a.datepicker._pos[1]+=b.offsetHeight}var l=false;a(b).parents().each(function(){l=a(this).css(\"position\")==\"fixed\";return!l});if(l&&a.browser.opera){a.datepicker._pos[0]-=document.documentElement.scrollLeft;a.datepicker._pos[1]-=document.documentElement.scrollTop}j={left:a.datepicker._pos[0],top:a.datepicker._pos[1]};a.datepicker._pos=null;f.dpDiv.empty();f.dpDiv.css({position:\"absolute\",display:\"block\",top:\"-1000px\"});a.datepicker._updateDatepicker(f); j=a.datepicker._checkOffset(f,j,l);f.dpDiv.css({position:a.datepicker._inDialog&&a.blockUI?\"static\":l?\"fixed\":\"absolute\",display:\"none\",left:j.left+\"px\",top:j.top+\"px\"});if(!f.inline){j=a.datepicker._get(f,\"showAnim\");var o=a.datepicker._get(f,\"duration\"),n=function(){var k=f.dpDiv.find(\"iframe.ui-datepicker-cover\");if(k.length){var m=a.datepicker._getBorders(f.dpDiv);k.css({left:-m[0],top:-m[1],width:f.dpDiv.outerWidth(),height:f.dpDiv.outerHeight()})}};f.dpDiv.zIndex(a(b).zIndex()+1);a.datepicker._datepickerShowing= true;a.effects&&a.effects[j]?f.dpDiv.show(j,a.datepicker._get(f,\"showOptions\"),o,n):f.dpDiv[j\"show\"](j?o:null,n);if(!j!o)n();f.input.is(\":visible\")&&!f.input.is(\":disabled\")&&f.input.focus();a.datepicker._curInst=f}}}},_updateDatepicker:function(b){this.maxRows=4;var f=a.datepicker._getBorders(b.dpDiv);i=b;b.dpDiv.empty().append(this._generateHTML(b));var j=b.dpDiv.find(\"iframe.ui-datepicker-cover\");j.length&&j.css({left:-f[0],top:-f[1],width:b.dpDiv.outerWidth(),height:b.dpDiv.outerHeight()}); b.dpDiv.find(\".\"+this._dayOverClass+\" a\").mouseover();f=this._getNumberOfMonths(b);j=f[1];b.dpDiv.removeClass(\"ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4\").width(\"\");j>1&&b.dpDiv.addClass(\"ui-datepicker-multi-\"+j).css(\"width\",17*j+\"em\");b.dpDiv[(f[0]!=1f[1]!=1?\"add\":\"remove\")+\"Class\"](\"ui-datepicker-multi\");b.dpDiv[(this._get(b,\"isRTL\")?\"add\":\"remove\")+\"Class\"](\"ui-datepicker-rtl\");b==a.datepicker._curInst&&a.datepicker._datepickerShowing&&b.input&&b.input.is(\":visible\")&& !b.input.is(\":disabled\")&&b.input[0]!=document.activeElement&&b.input.focus();if(b.yearshtml){var l=b.yearshtml;setTimeout(function(){l===b.yearshtml&&b.yearshtml&&b.dpDiv.find(\"select.ui-datepicker-year:first\").replaceWith(b.yearshtml);l=b.yearshtml=null},0)}},_getBorders:function(b){var f=function(j){return{thin:1,medium:2,thick:3}[j]j};return[parseFloat(f(b.css(\"border-left-width\"))),parseFloat(f(b.css(\"border-top-width\")))]},_checkOffset:function(b,f,j){var l=b.dpDiv.outerWidth(),o=b.dpDiv.outerHeight(), n=b.input?b.input.outerWidth():0,k=b.input?b.input.outerHeight():0,m=document.documentElement.clientWidth+a(document).scrollLeft(),p=document.documentElement.clientHeight+a(document).scrollTop();f.left-=this._get(b,\"isRTL\")?l-n:0;f.left-=j&&f.left==b.input.offset().left?a(document).scrollLeft():0;f.top-=j&&f.top==b.input.offset().top+k?a(document).scrollTop():0;f.left-=Math.min(f.left,f.left+l>m&&m>l?Math.abs(f.left+l-m):0);f.top-=Math.min(f.top,f.top+o>p&&p>o?Math.abs(o+k):0);return f},_findPos:function(b){for(var f= this._get(this._getInst(b),\"isRTL\");b&&(b.type==\"hidden\"b.nodeType!=1a.expr.filters.hidden(b));)b=b[f?\"previousSibling\":\"nextSibling\"];b=a(b).offset();return[b.left,b.top]},_triggerOnClose:function(b){var f=this._get(b,\"onClose\");if(f)f.apply(b.input?b.input[0]:null,[b.input?b.input.val():\"\",b])},_hideDatepicker:function(b){var f=this._curInst;if(!(!fb&&f!=a.data(b,\"datepicker\")))if(this._datepickerShowing){b=this._get(f,\"showAnim\");var j=this._get(f,\"duration\"),l=function(){a.datepicker._tidyDialog(f); this._curInst=null};a.effects&&a.effects[b]?f.dpDiv.hide(b,a.datepicker._get(f,\"showOptions\"),j,l):f.dpDiv[b==\"slideDown\"?\"slideUp\":b==\"fadeIn\"?\"fadeOut\":\"hide\"](b?j:null,l);bl();a.datepicker._triggerOnClose(f);this._datepickerShowing=false;this._lastInput=null;if(this._inDialog){this._dialogInput.css({position:\"absolute\",left:\"0\",top:\"-100px\"});if(a.blockUI){a.unblockUI();a(\"body\").append(this.dpDiv)}}this._inDialog=false}},_tidyDialog:function(b){b.dpDiv.removeClass(this._dialogClass).unbind(\".ui-datepicker-calendar\")}, _checkExternalClick:function(b){if(a.datepicker._curInst){b=a(b.target);b[0].id!=a.datepicker._mainDivId&&b.parents(\"#\"+a.datepicker._mainDivId).length==0&&!b.hasClass(a.datepicker.markerClassName)&&!b.hasClass(a.datepicker._triggerClass)&&a.datepicker._datepickerShowing&&!(a.datepicker._inDialog&&a.blockUI)&&a.datepicker._hideDatepicker()}},_adjustDate:function(b,f,j){b=a(b);var l=this._getInst(b[0]);if(!this._isDisabledDatepicker(b[0])){this._adjustInstDate(l,f+(j==\"M\"?this._get(l,\"showCurrentAtPos\"): 0),j);this._updateDatepicker(l)}},_gotoToday:function(b){b=a(b);var f=this._getInst(b[0]);if(this._get(f,\"gotoCurrent\")&&f.currentDay){f.selectedDay=f.currentDay;f.drawMonth=f.selectedMonth=f.currentMonth;f.drawYear=f.selectedYear=f.currentYear}else{var j=new Date;f.selectedDay=j.getDate();f.drawMonth=f.selectedMonth=j.getMonth();f.drawYear=f.selectedYear=j.getFullYear()}this._notifyChange(f);this._adjustDate(b)},_selectMonthYear:function(b,f,j){b=a(b);var l=this._getInst(b[0]);l[\"selected\"+(j==\"M\"? \"Month\":\"Year\")]=l[\"draw\"+(j==\"M\"?\"Month\":\"Year\")]=parseInt(f.options[f.selectedIndex].value,10);this._notifyChange(l);this._adjustDate(b)},_selectDay:function(b,f,j,l){var o=a(b);if(!(a(l).hasClass(this._unselectableClass)this._isDisabledDatepicker(o[0]))){o=this._getInst(o[0]);o.selectedDay=o.currentDay=a(\"a\",l).html();o.selectedMonth=o.currentMonth=f;o.selectedYear=o.currentYear=j;this._selectDate(b,this._formatDate(o,o.currentDay,o.currentMonth,o.currentYear))}},_clearDate:function(b){b=a(b); this._getInst(b[0]);this._selectDate(b,\"\")},_selectDate:function(b,f){b=this._getInst(a(b)[0]);f=f!=null?f:this._formatDate(b);b.input&&b.input.val(f);this._updateAlternate(b);var j=this._get(b,\"onSelect\");if(j)j.apply(b.input?b.input[0]:null,[f,b]);else b.input&&b.input.trigger(\"change\");if(b.inline)this._updateDatepicker(b);else{this._hideDatepicker();this._lastInput=b.input[0];typeof b.input[0]!=\"object\"&&b.input.focus();this._lastInput=null}},_updateAlternate:function(b){var f=this._get(b,\"altField\"); if(f){var j=this._get(b,\"altFormat\")this._get(b,\"dateFormat\"),l=this._getDate(b),o=this.formatDate(j,l,this._getFormatConfig(b));a(f).each(function(){a(this).val(o)})}},noWeekends:function(b){b=b.getDay();return[b>0&&b<6,\"\"]},iso8601Week:function(b){b=new Date(b.getTime());b.setDate(b.getDate()+4-(b.getDay()7));var f=b.getTime();b.setMonth(0);b.setDate(1);return Math.floor(Math.round((f-b)/864E5)/7)+1},parseDate:function(b,f,j){if(b==nullf==null)throw\"Invalid arguments\";f=typeof f==\"object\"? f.toString():f+\"\";if(f==\"\")return null;var l=(j?j.shortYearCutoff:null)this._defaults.shortYearCutoff;l=typeof l!=\"string\"?l:(new Date).getFullYear()%100+parseInt(l,10);for(var o=(j?j.dayNamesShort:null)this._defaults.dayNamesShort,n=(j?j.dayNames:null)this._defaults.dayNames,k=(j?j.monthNamesShort:null)this._defaults.monthNamesShort,m=(j?j.monthNames:null)this._defaults.monthNames,p=j=-1,q=-1,s=-1,r=false,u=function(z){(z=H+1-1){p=1;q=s;do{l=this._getDaysInMonth(j,p-1);if(q<=l)break;p++;q-=l}while(1)}C=this._daylightSavingAdjust(new Date(j,p-1,q));if(C.getFullYear()!=jC.getMonth()+1!=pC.getDate()!=q)throw\"Invalid date\";return C},ATOM:\"yy-mm-dd\", COOKIE:\"D, dd M yy\",ISO_8601:\"yy-mm-dd\",RFC_822:\"D, d M y\",RFC_850:\"DD, dd-M-y\",RFC_1036:\"D, d M y\",RFC_1123:\"D, d M yy\",RFC_2822:\"D, d M yy\",RSS:\"D, d M y\",TICKS:\"!\",TIMESTAMP:\"@\",W3C:\"yy-mm-dd\",_ticksTo1970:(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*24*60*60*1E7,formatDate:function(b,f,j){if(!f)return\"\";var l=(j?j.dayNamesShort:null)this._defaults.dayNamesShort,o=(j?j.dayNames:null)this._defaults.dayNames,n=(j?j.monthNamesShort:null)this._defaults.monthNamesShort;j=(j?j.monthNames: null)this._defaults.monthNames;var k=function(u){(u=r+1 12?b.getHours()+2:0);return b},_setDate:function(b,f,j){var l=!f,o=b.selectedMonth,n=b.selectedYear;f=this._restrictMinMax(b,this._determineDate(b,f,new Date));b.selectedDay=b.currentDay=f.getDate();b.drawMonth=b.selectedMonth=b.currentMonth=f.getMonth();b.drawYear=b.selectedYear=b.currentYear=f.getFullYear();if((o!=b.selectedMonthn!=b.selectedYear)&&!j)this._notifyChange(b);this._adjustInstDate(b);if(b.input)b.input.val(l?\"\":this._formatDate(b))},_getDate:function(b){return!b.currentYearb.input&& b.input.val()==\"\"?null:this._daylightSavingAdjust(new Date(b.currentYear,b.currentMonth,b.currentDay))},_generateHTML:function(b){var f=new Date;f=this._daylightSavingAdjust(new Date(f.getFullYear(),f.getMonth(),f.getDate()));var j=this._get(b,\"isRTL\"),l=this._get(b,\"showButtonPanel\"),o=this._get(b,\"hideIfNoPrevNext\"),n=this._get(b,\"navigationAsDateFormat\"),k=this._getNumberOfMonths(b),m=this._get(b,\"showCurrentAtPos\"),p=this._get(b,\"stepMonths\"),q=k[0]!=1k[1]!=1,s=this._daylightSavingAdjust(!b.currentDay? new Date(9999,9,9):new Date(b.currentYear,b.currentMonth,b.currentDay)),r=this._getMinMaxDate(b,\"min\"),u=this._getMinMaxDate(b,\"max\");m=b.drawMonth-m;var v=b.drawYear;if(m<0){m+=12;v--}if(u){var w=this._daylightSavingAdjust(new Date(u.getFullYear(),u.getMonth()-k[0]*k[1]+1,u.getDate()));for(w=r&&ww;){m--;if(m<0){m=11;v--}}}b.drawMonth=m;b.drawYear=v;w=this._get(b,\"prevText\");w=!n?w:this.formatDate(w,this._daylightSavingAdjust(new Date(v,m-p,1)),this._getFormatConfig(b)); w=this._canAdjustMonth(b,-1,v,m)?\\' \\'+w+\" \":o?\"\":\\' \\'+w+\" \";var x=this._get(b,\"nextText\");x=!n?x:this.formatDate(x,this._daylightSavingAdjust(new Date(v, m+p,1)),this._getFormatConfig(b));o=this._canAdjustMonth(b,+1,v,m)?\\' \\'+x+\" \":o?\"\":\\' \\'+x+\" \";p=this._get(b,\"currentText\");x=this._get(b,\"gotoCurrent\")&& b.currentDay?s:f;p=!n?p:this.formatDate(p,x,this._getFormatConfig(b));n=!b.inline?\\'\\'+this._get(b,\"closeText\")+\"\":\"\";l=l?\\'\\n')\n",
            "Smallest ratio: ('????[/b][/url], [url=http://www.moofpack.com/???????-??-gucci-???????-?????-????????-????-203550aa61g2019-p-3264.html][b]???????\\n', '.\\n')\n",
            "Biggest ratio: ('-\\n', 'Cамые свежие новости о главных событиях власти,политики,спорта,культуры,происшествия,экономики,бизнеса,иносми -\\n')\n",
            "Counts of pairs: 12061155\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def plot_len_hist(stat):\n",
        "    plt.hist(stat[1]['len']['eng'], 40, facecolor='g')#[x[0] for x in distr['eng']], 40, facecolor='g')\n",
        "    plt.xlabel('Length of sentence, charecters')\n",
        "    plt.ylabel('Amount of senteces with such length')\n",
        "    plt.show()\n",
        "\n",
        "plot_len_hist(stats_init)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "Y-dBOwHdogC6",
        "outputId": "053c93c9-88cc-4100-dcc8-ba2604750c60"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAERCAYAAACKHYuuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgdVZ3/8feHQEDZQyICARIw4ARFwJZFGA0BFVDAUZBFGREERdlnGEH8AeLMPIDjjOCwhUUE2RUxYiAihEXZ0oEESDQQA0oQh4jsPAKB7++Pc5pULt33Vne6ble6P6/nuU+qTm3fPr18U3VOnaOIwMzMrJnlBjoAMzOrPycLMzNrycnCzMxacrIwM7OWnCzMzKwlJwszM2tpmUwWki6W9LSkh0vs+z+SZubPI5Kea0eMZmaDiZbF9ywkfQR4Cbg0It7Xi+OOALaMiIMqC87MbBBaJu8sIuIO4G/FMkkbS7pJ0gxJd0p6bzeH7gdc2ZYgzcwGkeUHOoB+NAn4akQ8Kmkb4BxgYtdGSRsCY4FbByg+M7Nl1qBIFpJWAT4MXCupq3jFht32BX4SEW+0MzYzs8FgUCQL0uO05yJiiyb77At8vU3xmJkNKstkm0WjiHgBeEzS3gBKPtC1PbdfrAncPUAhmpkt05bJZCHpStIf/k0lLZB0MPB54GBJs4DZwJ6FQ/YFroplseuXmVkNLJNdZ83MrL2WyTsLMzNrr2WugXvkyJExZsyYgQ7DzGyZMmPGjL9GxKi+Hr/MJYsxY8bQ2dk50GGYmS1TJP1xaY73YygzM2vJycLMzFpysjAzs5acLMzMrKXKkkWrOSckfV7Sg5IeknRX8Y1rMzOrlyrvLC4Bdmmy/THgoxHxfuA7pFFjzcyshirrOhsRd0ga02T7XYXVe4DRVcViZmZLpy5tFgcDN/a0UdKhkjoldS5cuLCNYZmZGdQgWUjakZQsvtHTPhExKSI6IqJj1Kg+v4BoZmZ9NKBvcEvaHLgQ2DUinqn8et9W0+1xsgdVNDPrzoDdWUjaALgOOCAiHhmoOMzMrLXK7izynBMTgJGSFgAnAysARMR5wEnAWsA5eSrURRHRUVU8ZmbWd1X2htqvxfYvA1+u6vpmZtZ/BryB28zM6s/JwszMWnKyMDOzlpwszMyspZYN3JK2B04BNsz7C4iI2Kja0MzMrC7K9Ia6CDgGmAG8UW04ZmZWR2WSxfMR0eO4TWZmNvj1mCwkbZUXp0n6Lult61e7tkfE/RXHZmZmNdHszuJ7DevFt6sDmNj/4ZiZWR31mCwiYkcASRtFxPziNklu3DYzG0LKdJ39STdl1/Z3IGZmVl/N2izeC2wGrC7pM4VNqwErVR2YmZnVR7M2i02BTwFrALsXyl8EDqkyKDMzq5dmbRY/B34uabuIuLuNMZmZWc2Uec9if0mNw40/D3TmhGJmZoNcmQbuFYEtgEfzZ3NgNHCwpO9XGJuZmdVEmTuLzYHtI+INAEnnAncCOwAPVRibmZnVRJk7izWBVQrrKwMjcvJ4tftDzMxsMClzZ3EGMFPSbaQRZz8C/KeklYFfVxibmZnVRMtkEREXSZoCbJ2LvhkRf87Lx1UWmZmZ1UbZyY+WAxYCzwLvkfSR6kIyM7O6KTP50enAPsBs4M1cHMAdFcZlZmY1UqbN4tPAphHhxmwzsyGqzGOo+cAKVQdiZmb1VebO4hVSb6hbWHLyoyMri8rMzGqlTLKYnD9mZjZElek6+yNJ7wA2iIi5ZU8s6WLSqLVPR8T7utku4ExgN9Ldy4GeqtXMrJ5atllI2h2YCdyU17eQVOZO4xJglybbdwXG5c+hwLklzmlmZgOgTAP3KaQX8p4DiIiZQMtpVSPiDuBvTXbZE7g0knuANSStUyIeMzNrszLJ4vWIeL6h7M1u9+yd9YAnCusLctnbSDpUUqekzoULF/bDpc3MrDfKJIvZkvYHhkkaJ+kHwF0Vx7WEiJgUER0R0TFq1Kh2XtrMzCiXLI4gzcX9KnAl8AJwdD9c+0lg/cL66FxmZmY1U6Y31CvAifnTnyYDh0u6CtgGeD4inurna5iZWT/oMVlI+gVpDKhuRcQezU4s6UpgAjBS0gLgZPKb4BFxHjCF1G12Hqnr7Jd6GbuZmbVJszuL/1qaE0dE47zdjdsD+PrSXMPMzNqjx2QREbe3MxAzM6uvsvNZmJnZEOZkYWZmLTlZmJlZS2VmytuENNf2hsX9I2JihXGZmVmNlBmi/FrgPOAC4I1qwzEzszoqkywWRYRHhDUzG8KavZQ3Ii/+QtLXgJ+x5Ex5zUaUNTOzQaTZncUM0hvcyuvHFbYFJYYpNzOzwaHZS3lj2xmImZnVV5mZ8r4uaY3C+pr5sZSZmQ0RZd6zOCQinutaiYhngUOqC8nMzOqmTLIYJqmr3QJJw4Dh1YVkZmZ1U6br7E3A1ZLOz+tfyWVmZjZElEkW3yAliMPy+s3AhZVFZGZmtVNmprw3gXPzx8zMhqAyY0M9Rjcz5kWE37MwMxsiyjyG6igsrwTsDYzoYV8zMxuEWvaGiohnCp8nI+L7wCfbEJuZmdVEmcdQWxVWlyPdaZS5IzEzs0GizB/97xWWFwGPA5+rJBozM6ulMr2hdmxHIGZmVl9lxoY6StJqSi6UdL+kj7cjODMzq4cyw30cFBEvAB8H1gIOAE6rNCozM6uVMsmia1yo3YBLI2J2oczMzIaAMslihqRfkZLFVEmrAm+WObmkXSTNlTRP0vHdbN9A0jRJD0h6UNJuvQvfzMzaoUxvqIOBLYD5EfGKpLWAL7U6KI9OezbwMWABMF3S5IiYU9jtW8A1EXGupPHAFGBML78GMzOrWNmxoe4vrD8DPFPi3FsD8yJiPoCkq4A9gWKyCGC1vLw68OdyYZuZWTtV+XLdesAThfUFwDYN+5wC/ErSEcDKwM4VxmNmZn1Ups2iSvsBl0TEaFKbyGWS3haTpEMldUrqXLhwYduDNDMb6kolC0nDJK2bG6Q3kLRBicOeBNYvrI/OZUUHA9cARMTdpIEKRzaeKCImRURHRHSMGjWqTMhmZtaPyryUdwTwf6RJj36ZPzeUOPd0YJyksZKGA/sCkxv2+ROwU77OP5CShW8dzMxqpkybxVHAprlhu7SIWCTpcGAqMAy4OCJmSzoV6IyIycC/ABdIOobU2H1gRLxt7gwzMxtYZZLFE8DzfTl5REwhdYctlp1UWJ4DbN+Xc5uZWfv0mCwkHZsX5wO3Sfol8GrX9oj474pjMzOzmmh2Z7Fq/vdP+TM8f6CbaVbNzGzw6jFZRMS3ASTtHRHXFrdJ2rvqwMzMrD7KdJ09oWSZmZkNUs3aLHYlvSi3nqSzCptWI82YZ2ZmQ0SzNos/A53AHsCMQvmLwDFVBmVmZvXSrM1iFjBL0uUR4TsJM7MhrNljqGsi4nPAA5Le1vspIjavNDIzM6uNZo+hjsr/fqodgZiZWX01ewz1VF7cGbgjIh5tT0hmZlY3ZYb72AA4X9IYUkP3HcCdETGzwrjMzKxGWr5nEREnR8REYDPgTuA4luwdZWZmg1zLOwtJ3yIN9rcK8ADwr6SkYWZmQ0SZx1CfIb2E90vgduDuiHi1+SFmZjaYlHkMtRWpkfs+4GPAQ5J+U3VgZmZWH2UeQ70P+Efgo0AHaX4LP4YyMxtCyjyGOo3UA+osYHpEvF5tSGZmVjctk0VE+KU8M7MhrswQ5WZmNsQ5WZiZWUu9ShaSlpO0WlXBmJlZPbVMFpKukLSapJWBh4E5ko6rPjQzM6uLMncW4yPiBeDTwI3AWOCASqMyM7NaKZMsVpC0AilZTM5dZ982v4WZmQ1eZZLF+cDjwMrAHZI2BF6oMigzM6uXMu9ZnEV6Ia/LHyXtWF1IZmZWN2UauNeWdJGkG/P6eOCLZU4uaRdJcyXNk3R8D/t8TtIcSbMlXdGr6M3MrC3KPIa6BJgKrJvXHwGObnWQpGHA2cCuwHhgv5xoivuMA04Ato+Izcqc18zM2q9MshgZEdcAbwJExCLgjRLHbQ3Mi4j5EfEacBWwZ8M+hwBnR8Sz+dxPl47czMzapkyyeFnSWuQeUJK2BZ4vcdx6pBFquyzIZUWbAJtI+q2keyTt0t2JJB0qqVNS58KFC0tc2szM+lOZUWePBSYDG0v6LTAK2Ksfrz8OmACMJvW2en9EPFfcKSImAZMAOjo63G3XzKzNyvSGul/SR4FNAQFzSw5T/iSwfmF9dC4rWgDcm8/3mKRHSMljepngzcysPcr0hvo6sEpEzI6Ih4FVJH2txLmnA+MkjZU0HNiXdIdSdD3prgJJI0mPpeb3In4zM2uDMm0WhxQfC+XG6ENaHZQbwg8n9aT6HXBNRMyWdKqkPfJuU4FnJM0BpgHHRcQzvf0izMysWmXaLIZJUkR0NXAPA4aXOXlETAGmNJSdVFgOUpvIsaUjNjOztiuTLG4CrpZ0fl7/Si4zM7Mhokyy+AYpQRyW128GLqwsIjMzq50yvaHelHQJcGtEzK0+JDMzq5syvaH2AGaSHz1J2kJSY68mMzMbxMr0hjqZNHTHcwARMZM0AZKZmQ0RZZLF6xHROLyH36I2MxtCyjRwz5a0P6kL7TjgSOCuasMyM7M6KXNncQSwGfAqcAVpEMGjqgzKzMzqpcydxScj4kTgxK4CSXsD11YWlZmZ1UqZO4sTSpaZmdkg1eOdhaRdgd2A9SQV5+BeDVhUdWBmZlYfzR5D/RnoBPYAZhTKXwSOqTIoMzOrlx6TRUTMAmZJuqLk/BVmZjZIlWng3lrSKcCGeX+RBozdqMrAzMysPsoki4tIj51mAG9UG46ZmdVRmWTxfETcWHkkZmZWW2WSxTRJ3wWuI72YB6S5uSuLyszMaqVMstgm/9tRKAtgYv+HY2ZmdVRmPosd2xGImZnVV5n5LNaWdJGkG/P6eEkHVx+amZnVRZnhPi4BpgLr5vVHgKOrCsjMzOqnTLIYGRHXAG8CRMQi3IXWzGxIKZMsXpa0FnnCI0nbkoYpNzOzIaJMb6hjgcnAxpJ+C4wC9qo0KjMzq5UyvaHul/RRYFPSUB9zPVaUmdnQUqY31N7AOyJiNvBp4GpJW1UemZmZ1UaZNov/FxEvStoB2Ik0VtS5ZU4uaRdJcyXNk3R8k/0+KykkdfS0j5mZDZwyyaKr59MngQsi4pfA8FYHSRoGnA3sCowH9pM0vpv9ViXN6X1v2aDNzKy9yiSLJyWdD+wDTJG0YsnjtgbmRcT8iHgNuArYs5v9vgOcDvy9ZMxmZtZmZf7of470Ut4nIuI5YARwXInj1gOeKKwvyGVvyW0f6+e7lR5JOlRSp6TOhQsXlri0mZn1pzK9oV4hjTjbtf4U8NTSXljScsB/AweWiGESMAmgo6MjlvbaZmbWO2XuLPrqSWD9wvroXNZlVeB9wG2SHge2BSa7kdvMrH56TBa5bWJpTAfGSRoraTiwL+nlPgAi4vmIGBkRYyJiDHAPsEdEdC7ldc3MrJ81u7O4G0DSZX05cR5D6nBSe8fvgGsiYrakUyXt0ZdzmpnZwGjWZjFc0v7AhyV9pnFjRFzXzTGN+0wBpjSUndTDvhNanc/MzAZGs2TxVeDzwBrA7g3bgkKjt5mZDW49JouI+A3wG0mdEXFRG2MyM7OaKTPq7GWSjgQ+ktdvB87zYIJmZkNHmWRxDrBC/hfgANLYUF+uKigzM6uXMsniQxHxgcL6rZJmVRWQmZnVT6mBBCVt3LUiaSM8raqZ2ZBS5s7iOGCapPmkyY82BL5UaVRmZlYrZcaGukXSONJMeZBmynu12rDMzKxOytxZkJPDgxXHYmZmNVXlQIJmZjZIOFmYmVlLLZOFpFvKlJmZ2eDVY5uFpJWAdwIjJa1J6gkFsBoNM96Zmdng1qyB+yvA0cC6wAwWJ4sXgP+tOC4zM6uRZgMJngmcKemIiPhBG2MyM7OaKfOexQ8kfRgYU9w/Ii6tMC4zM6uRlskiz5S3MTCTxcN8BOBkYWY2RJR5Ka8DGB8RUXUwZmZWT2Xes3gYeHfVgZiZWX2VubMYCcyRdB/w1phQEbFHZVGZmVmtlEkWp1QdhJmZ1VuZ3lC3tyMQMzOrrzK9oV4k9X4CGE6aYvXliFitysDMzKw+ytxZrNq1LEnAnsC2VQZlZmb10qtRZyO5HvhERfGYmVkNlXkM9ZnC6nKk9y7+XllEZmZWO2V6Q+1eWF4EPE56FNWSpF2AM4FhwIURcVrD9mOBL+fzLgQOiog/ljm3mZm1T5k2iy/15cSShgFnAx8DFgDTJU2OiDmF3R4AOiLiFUmHAWcA+/TlemZmVp0ykx+NlvQzSU/nz08ljS5x7q2BeRExPyJeA66i4Y4kIqZFxCt59R6gzHnNzKzNyjRw/xCYTJrXYl3gF7mslfWAJwrrC2g+adLBwI3dbZB0qKROSZ0LFy4scWkzM+tPZZLFqIj4YUQsyp9LgFH9GYSkL5Aazr/b3faImBQRHRHRMWpUv17azMxKKJMsnpH0BUnD8ucLwDMljnsSWL+wPjqXLUHSzsCJwB4R8WrjdjMzG3hlksVBwOeAvwBPAXsBZRq9pwPjJI2VNBzYl/Q46y2StgTOJyWKp3sTuJmZtU+Z3lB/BHo9wmxELJJ0ODCV1HX24oiYLelUoDMiJpMeO60CXJteDudPHs3WzKx+yryUNxY4grdPq9ryj3pETAGmNJSdVFjeuRexmpnZACnzUt71wEWkXlBvVhuOmZnVUZlk8feIOKvySMzMrLbKJIszJZ0M/IolZ8q7v7KozMysVsoki/cDBwATWfwYKvK6mZkNAWWSxd7ARnnIDjMzG4LKvGfxMLBG1YGYmVl9lbmzWAP4vaTpLNlm4fchzMyGiDLJ4uTKozAzs1or8wb37cV1STsA+wG3d3+EmZkNNmXuLLrGcNqf1Nj9GPDTKoMyM7N66TFZSNqEdAexH/BX4GpAEbFjm2IzM7OaaHZn8XvgTuBTETEPQNIxbYnKzMxqpVnX2c+QhiSfJukCSTsBak9YZmZWJz0mi4i4PiL2Bd4LTAOOBt4l6VxJH29XgGZmNvBavpQXES9HxBURsTtptrsHgG9UHpmZmdVGmTe43xIRz+b5sHeqKiAzM6ufXiULMzMbmpwszMysJScLMzNrycnCzMxacrIwM7OWnCzMzKwlJwszM2vJycLMzFpysjAzs5acLMzMrKVSkx/1laRdgDOBYcCFEXFaw/YVgUuBDwLPAPtExONVxtSMvt18UN04OdoUiZlZvVR2ZyFpGHA2sCswHthP0viG3Q4Gno2I9wD/A5xeVTxmZtZ3VT6G2hqYFxHzI+I14Cpgz4Z99gR+lJd/AuwkyXNmmJnVTJWPodYDniisLwC26WmfiFgk6XlgLdI0rm+RdChwaF59SdLcPsY0svHcvaFTKs9jSxVfxRxb3zi2vnFsfddTfBsuzUkrbbPoLxExCZi0tOeR1BkRHf0QUiXqHJ9j6xvH1jeOre+qiq/Kx1BPAusX1kfnsm73kbQ8sDqpodvMzGqkymQxHRgnaayk4cC+wOSGfSYDX8zLewG3RoS7HJmZ1Uxlj6FyG8ThwFRS19mLI2K2pFOBzoiYDFwEXCZpHvA3UkKp0lI/yqpYneNzbH3j2PrGsfVdJfHJ/5E3M7NW/Aa3mZm15GRhZmYtDZlkIWkXSXMlzZN0fJuuub6kaZLmSJot6ahcPkLSzZIezf+umcsl6awc44OStiqc64t5/0clfbGna/YhxmGSHpB0Q14fK+neHMPVuXMCklbM6/Py9jGFc5yQy+dK+kQ/xbWGpJ9I+r2k30nari71JumY/P18WNKVklYayHqTdLGkpyU9XCjrt7qS9EFJD+VjzpLKvzjbQ2zfzd/XByX9TNIareqkp9/fnuq9r7EVtv2LpJA0si71lsuPyHU3W9IZhfLq6y0iBv2H1MD+B2AjYDgwCxjfhuuuA2yVl1cFHiENfXIGcHwuPx44PS/vBtwICNgWuDeXjwDm53/XzMtr9lOMxwJXADfk9WuAffPyecBheflrwHl5eV/g6rw8PtfnisDYXM/D+iGuHwFfzsvDgTXqUG+kF0kfA95RqK8DB7LegI8AWwEPF8r6ra6A+/K+ysfuupSxfRxYPi+fXoit2zqhye9vT/Xe19hy+fqkjjl/BEbWqN52BH4NrJjX39XOeqv0j2VdPsB2wNTC+gnACQMQx8+BjwFzgXVy2TrA3Lx8PrBfYf+5eft+wPmF8iX2W4p4RgO3ABOBG/IP9V8Lv8hv1Vv+5dkuLy+f91NjXRb3W4q4Vif9QVZD+YDXG4tHHRiR6+EG4BMDXW/AmIY/LP1SV3nb7wvlS+zXl9gatv0TcHle7rZO6OH3t9nP69LERhp66APA4yxOFgNeb6Q/8Dt3s19b6m2oPIbqbuiR9doZQH78sCVwL7B2RDyVN/0FWDsv9xRnVfF/H/g34M28vhbwXEQs6uY6SwzNAnQNzVJFbGOBhcAPlR6RXShpZWpQbxHxJPBfwJ+Ap0j1MIN61FtRf9XVenm5qjgPIv2vuy+xNft57RNJewJPRsSshk11qLdNgH/Mj49ul/ShPsbWp3obKsliQElaBfgpcHREvFDcFim1t73/sqRPAU9HxIx2X7uE5Um34OdGxJbAy6RHKW8ZwHpbkzQA5lhgXWBlYJd2x9EbA1VXrUg6EVgEXD7QsQBIeifwTeCkgY6lB8uT7mi3BY4DrulNO8jSGirJoszQI5WQtAIpUVweEdfl4v+TtE7evg7wdIs4q4h/e2APSY+TRgSeSJp7ZA2loVcar9PT0CxVxLYAWBAR9+b1n5CSRx3qbWfgsYhYGBGvA9eR6rIO9VbUX3X1ZF7u1zglHQh8Cvh8TmZ9ie0Zeq73vtiY9J+AWfn3YjRwv6R39yG2KuptAXBdJPeRngiM7ENsfau3vjwfXdY+pIw8n/SD0NXQs1kbrivS5E7fbyj/Lks2Pp6Rlz/Jko1o9+XyEaRn+Gvmz2PAiH6McwKLG7ivZcmGr6/l5a+zZEPtNXl5M5ZsXJtP/zRw3wlsmpdPyXU24PVGGjl5NvDOfL0fAUcMdL3x9ufb/VZXvL2hdreljG0XYA4wqmG/buuEJr+/PdV7X2Nr2PY4i9ss6lBvXwVOzcubkB4xqV31VskfyTp+SL0ZHiH1DjixTdfcgXT7/yAwM392Iz0zvAV4lNS7oeuHS6QJo/4APAR0FM51EDAvf77Uz3FOYHGy2Cj/kM/LP1BdPS9Wyuvz8vaNCsefmGOeSy96fLSIaQugM9fd9fkXsRb1Bnwb+D3wMHBZ/iUdsHoDriS1n7xO+t/nwf1ZV0BH/lr/APwvDR0P+hDbPNIfuq7fifNa1Qk9/P72VO99ja1h++MsThZ1qLfhwI/zOe8HJraz3jzch5mZtTRU2izMzGwpOFmYmVlLThZmZtaSk4WZmbXkZGFmZi05WViPJL1U8fmPzm/NLvX1lEZ3/bWkmZL26Z8I33aNb1Zx3v4gaYLyyMFtut4YSfu363o28JwsbCAdTXq5rT9sCRARW0TE1f10zka1TRZLq/A2b1ljgF4liz5cw2rEycJ6RdLGkm6SNEPSnZLem8svyWP23yVpvqS9cvlyks7JY/DfLGmKpL0kHUkaW2mapGmF8/+HpFmS7pG0djfXHyHp+jynwD2SNpf0LtLLSh/KdxYbNxxzpNKcIg9KuiqXrZznDLgvD1a4Zy4/UNJ1+Wt8tGvOAEmnAe/I5788l30hHz9T0vmShuXyl7r7OiStrTR/w6z8+XCz8zT5Hrwn30XNknR/4etdRYvnALm8a9wgSSdJmq40/8akQvltkr4vqRM4Smn+hdvz93ZqYbiQ7q53GmlQu5lK83sMU5qnYnqu56/kYyfkn5PJwJxc77/M53q4qrtAq0B/vNHqz+D8AC91U3YLMC4vbwPcmpcvIb0JuhxpfP15uXwvYEoufzfwLLBX3vY4+Q3ZvB7A7nn5DOBb3Vz/B8DJeXkiMDMvTyC/hd7NMX9m8RvVa+R//xP4QlcZ6S3XlUlzU8wnjeG0EmlOg/Ub6wP4B+AXwAp5/Rzgn5t9HcDVpMEkIQ3HsHqz8zT5vtwL/FNeXol0dzaBNKLt6FzXdwM75H1GFI69rBDbbcA5eXkF4C7yEBzAPsDFLa53Q+G8hxa+zhVJb9+Pzfu9DIzN2z4LXFA4bvWB/jn3p9zHt4VWmtLouR8GrtXiwS5XLOxyfUS8SfofZNddwQ7Atbn8L8W7iG68RpofAtKw3x/rZp8dSH9wiIhbJa0labUWoT8IXC7petLQIZAm4NlD0r/m9ZWADfLyLRHxfP6a5wAbsuRQzwA7AR8Epue6eAeLB+vr6euYCPxzjv0N4HlJBzQ5z9tIWhVYLyJ+ls/z91wOabyiBXl9JulR0W+AHSX9G+mP/AjS2Fa/yKfsemS3KfA+4OZ8rmHAUy2uV/RxYPOuO0pSIhyX6+K+iHgslz8EfE/S6aRkc2dPX6vVi5OF9cZypHHwt+hh+6uF5b4Mnfx65P9uAm/Qfz+fnyTNPLY7cKKk9+f4PhsRc4s7StqGJb+OnuIQ8KOIOKGbbb35Opqdp7feFreklUh3Kx0R8YSkU0iJscvLhThmR8R2SwSXkkUZAo6IiKkNx08oXIOIeERpStLdgH+XdEtEnFryGjaA3GZhpUWai+MxSXvDW/MSf6DFYb8FPqvUdrE26bFElxdJ0832xp3A5/P1JwB/jYY5QookLUd6jDQN+Abpf7yrkGYTO6Lw/H7LEtd+XWnIeUiP4/ZSai/pakvZsMXxtwCH5f2HSVq92XkkXSpp6+IJIuJFYIGkT+d9VlShR1k3uhLDX/Od4V497DcXGCVpu3zeFSRt1uR6jd+7qcBhXfUjaROlCauWIGld4JWI+DFpZNytGvexenKysGbeKWlB4XMs6Q/1wZJmkR5n7NniHD8ljZo5h9QIfT/p2TrAJOCmFo+mGp0CfFDSg6RG1i+22H8Y8GNJDwEPAGdFxHPAd0jP6R+UNDuvtzIp7395RMwBvgX8KsdyM2kqzWaOIj0Seoj0eGp8i/NsTmpvaXQAcGTe/y5SW1C38td6AWmk0qnA9B72e42USE7P39uZpEeOPV3vQeCN3B6y0QYAAACASURBVFB9DHAh6Xt8v6SHSdOLdndH9X7gvvyY7GTg33uK3erFo85a5SStEhEvSVqLNCzy9hHxl4GOq85yO8xFEbH3QMdiBk4W1gaSbiP1OBpOmoTnkgENyMx6zcnCzMxacpuFmZm15GRhZmYtOVmYmVlLThZmZtaSk4WZmbX0/wGCGnXnYLOPCAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_counts_boxplot(stats):\n",
        "    l = stats[1]['len']\n",
        "    plt.boxplot(l['eng'])\n",
        "    #plt.boxplot(l['rus'])\n",
        "    plt.ylabel(\"Length of Eng sentence\")\n",
        "    plt.show()\n",
        "\n",
        "plot_counts_boxplot(stats_init)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "_sKmkDWfryXF",
        "outputId": "80a4b8a9-e099-4b47-ac62-a394f6212cf1"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAD4CAYAAAAtrdtxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAdoElEQVR4nO3df3RfdZ3n8eeLNBBgwLRQGbc/aMWiwYzOYAZQu3OMzkBx0OJvcjxjlRyqK0ZndVascbai5oDsrGzpWKAaBMQNv8aByg+xi0E3q/xI0YFCxqVLB0mP0kL54eK2Tcp7/7if4LchP25v8/1+8yWvxzn35N73/fX+nmN5e+/ncz8fRQRmZmZFHFTtBMzMrHa5iJiZWWEuImZmVpiLiJmZFeYiYmZmhc2qdgKVdvTRR8eiRYuqnYaZWU3ZtGnTkxExd3S8bEVE0hXAGcD2iGguiXcA5wJ7gVsj4vMpvgpoT/FPR8QdKb4MWAPUAd+OiAtTfDFwLXAUsAn4m4jYM1leixYtor+/f8p+p5nZTCDpsbHi5XyddSWwbFQSrcBy4I0R8XrgH1L8BOAs4PXpnHWS6iTVAd8ETgdOANrSsQBfBy6OiNcAT5MVIDMzq6CyFZGI+Cmwc1T4PwAXRsTudMz2FF8OXBsRuyNiK7AFOCktWyLi0fSUcS2wXJKAtwM3pvOvAs4s128xM7OxVbph/Xjg30u6R9JPJP15is8DHi85bjDFxosfBTwTEcOj4mZmVkGVblifBcwBTgH+HLhe0qvLfVNJK4GVAAsXLiz37czMZoxKP4kMAt+PzL3AC8DRwDZgQclx81NsvPhTQKOkWaPiY4qI9RHREhEtc+e+pHOBmZkVVOkichPQCiDpeOBg4ElgA3CWpENSr6slwL3AfcASSYslHUzW+L4hslEje4H3p+uuAG6u6C8xmyI9PT00NzdTV1dHc3MzPT091U7JLLdydvHtAd4GHC1pEFgNXAFcIWkzsAdYkQrCQ5KuBx4GhoFzI2Jvus6ngDvIuvheEREPpVucB1wr6WvAL4Ducv0Ws3Lp6emhs7OT7u5uli5dSl9fH+3tWUfDtra2KmdnNjnNtKHgW1pawt+J2HTR3NzM2rVraW1tfTHW29tLR0cHmzdvrmJmZvuStCkiWl4SdxExq566ujp27dpFfX39i7GhoSEaGhrYu3dvFTMz29d4RcRjZ5lVUVNTE319ffvE+vr6aGpqqlJGZvvHRcSsijo7O2lvb6e3t5ehoSF6e3tpb2+ns7Oz2qmZ5TLjBmA0m05GGs87OjoYGBigqamJrq4uN6pbzXCbiJmZTcptImZmNuVcRMzMrDAXETMzK8xFxMzMCnMRMTOzwlxEzMysMBcRMzMrzEXEzMwKcxExM7PCXETMzKwwFxEzMyvMRcTMzAorWxGRdIWk7Wkq3NH7PicpJB2dtiXpEklbJD0g6cSSY1dIeiQtK0rib5L0YDrnEkkq128xM7OxlfNJ5Epg2eigpAXAqcCvS8KnA0vSshK4NB07h2xu9pOBk4DVkmancy4Fzik57yX3MjOz8ipbEYmInwI7x9h1MfB5oHQM+uXA1ZG5G2iU9CrgNGBjROyMiKeBjcCytO/IiLg7srHsrwbOLNdvMTOzsVW0TUTScmBbRPzLqF3zgMdLtgdTbKL44Bjx8e67UlK/pP4dO3YcwC8wM7NSFSsikg4Dvgj850rdc0RErI+IlohomTt3bqVvb2b2slXJJ5HjgMXAv0j6N2A+cL+kPwa2AQtKjp2fYhPF548RNzOzCqpYEYmIByPilRGxKCIWkb2COjEifgtsAD6SemmdAjwbEb8B7gBOlTQ7NaifCtyR9j0n6ZTUK+sjwM2V+i1mZpYpZxffHuDnwGslDUpqn+Dw24BHgS3At4BPAkTETuCrwH1p+UqKkY75djrn/wC3l+N3mJnZ+JR1bpo5Wlpaor+/v9ppmJnVFEmbIqJldNxfrJuZWWEuImZmVpiLiJmZFeYiYmZmhbmImJlZYS4iZmZWmIuImZkV5iJiZmaFuYiYmVlhLiJmZlaYi4iZmRXmImJmZoW5iJiZWWEuImZmVpiLiFmV9fT00NzcTF1dHc3NzfT09FQ7JbPcZlU7AbOZrKenh87OTrq7u1m6dCl9fX20t2fzt7W1tVU5O7PJTfokIukwSX8v6Vtpe4mkM3Kcd4Wk7ZI2l8T+i6R/lfSApH+W1Fiyb5WkLZJ+Jem0kviyFNsi6Qsl8cWS7knx6yQdvD8/3Gw66Orqoru7m9bWVurr62ltbaW7u5uurq5qp2aWS57XWd8BdgNvTtvbgK/lOO9KYNmo2EagOSLeAPxvYBWApBOAs4DXp3PWSaqTVAd8EzgdOAFoS8cCfB24OCJeAzwNTDT9rtm0NDAwwNKlS/eJLV26lIGBgSplZLZ/8hSR4yLiImAIICJ+D2iykyLip8DOUbEfRcRw2rwbmJ/WlwPXRsTuiNhKNm/6SWnZEhGPRsQe4FpguSQBbwduTOdfBZyZ47eYTStNTU309fXtE+vr66OpqalKGZntnzxFZI+kQ4EAkHQc2ZPJgTobuD2tzwMeL9k3mGLjxY8CnikpSCPxMUlaKalfUv+OHTumIHWzqdHZ2Ul7ezu9vb0MDQ3R29tLe3s7nZ2d1U7NLJc8DeurgR8CCyR9D3gr8NEDuamkTmAY+N6BXCeviFgPrAdoaWmJStzTLI+RxvOOjg4GBgZoamqiq6vLjepWMyYtIhGxUdL9wClkr7E+ExFPFr2hpI8CZwDviIiR/6BvAxaUHDY/xRgn/hTQKGlWehopPd6sprS1tbloWM3K0zvrPcBwRNwaEbcAw5IKtT9IWgZ8Hnh3alsZsQE4S9IhkhYDS4B7gfuAJakn1sFkje8bUvHpBd6fzl8B3FwkJzMzKy5Pm8jqiHh2ZCMiniF7xTUhST3Az4HXShqU1A78I3AEsFHSLyVdlq75EHA98DDZq7NzI2Jvesr4FHAHMABcn44FOA/4rKQtZG0k3bl+sZmZTRn94Y3SOAdID6QuuaWxByPiT8qaWZm0tLREf39/tdMwM6spkjZFRMvoeJ4nkX5J35B0XFq+AWya+hTNzKzW5CkiHcAe4Lq07AbOLWdSZmZWG/L0znoe+MJkx5mZ2cwzaRGRdDzwd8Ci0uMj4u3lS8vMzGpBno8NbwAuA74N7C1vOmZmVkvytIkMR8SlEXFvRGwaWcqemdkM4flErJbleRL5gaRPAv9MyZhZEbFz/FPMLA/PJ2K1Ls93IlvHCEdEvLo8KZWXvxOx6aS5uZm1a9fS2tr6Yqy3t5eOjg42b948wZlmlTXedyKTFpGXGxcRm07q6urYtWsX9fX1L8aGhoZoaGhg7143Qdr0UfhjwzSz4ZckrU/buWY2NLPJeT4Rq3V5ZzbcA7wlbeed2dDMJuH5RKzW5WlYPy4iPiSpDbKZDdPMgmZ2gDyfiNW6PEWkXDMbmhmeT8RqW54i8mVeOrPhx8qZlJmZ1YY8Y2f9SNImpmhmQzMze/nI0zvrzoh4amRmw4h4UtKdlUjOzMymt3GfRCQ1AIcBR0uaTfYUAnAkMK8CuZmZ2TQ30ZPIx8kmn3pd+juy3Ew2ze2EJF0habukzSWxOZI2Snok/Z2d4pJ0iaQtkh6QdGLJOSvS8Y9IWlESf5OkB9M5l7jHmJlZ5Y1bRCJiTUQsBv4uIl4dEYvT8saImLSIAFcCy0bFvgDcGRFLgDv5wzwlpwNL0rISuBSyokM2n/vJwEnA6pHCk445p+S80fcyM7Myy9OwvlbSW3jpfCJXT3LeTyUtGhVeDrwtrV8F3AWcl+JXRzYGy92SGiW9Kh27cWSwR0kbgWWS7gKOjIi7U/xq4Ezg9sl+j5mZTZ08k1J9FzgO+CV/mE8kgAmLyDiOiYjfpPXfAsek9XnA4yXHDabYRPHBMeLj/YaVZE84LFy4sEDaZmY2ljzfibQAJ8QUj9QYESGpIqM/RsR6YD1kAzBW4p5mZjNBnrGzNgN/PEX3eyK9piL93Z7i24AFJcfNT7GJ4vPHiJuZWQXlKSJHAw9LukPShpGl4P02ACM9rFaQ9fQaiX8k9dI6BXg2vfa6AzhV0uzUoH4qcEfa95ykU1KvrI+UXMvMzCok77An+01SD1nD+NGSBsl6WV0IXC+pHXgM+GA6/DbgncAW4PekYVUiYqekrwL3peO+UjKj4ifJeoAdStag7kZ1M7MKyzUplaRjgSUR8T8kHQbURcTvyp5dGXhSKjOz/Xcgk1KdA9wIXJ5C84CbpjY9MzOrRXnaRM4lG7n3OYCIeAR4ZTmTMjOz2pCniOyOiD0jG5JmkeYWMTOzmS1PEfmJpC8Ch0r6K+AG4AflTcvMzGpBniLyBWAH8CDZoIy3AV8qZ1JmM0lHRwcNDQ1IoqGhgY6OjmqnZJZbnrGzXgC+BXwrDYg4f6q/XjebqTo6Oli3bh1z585l+/btNDY2sm7dOgDWrl1b5ezMJpend9Zdko5MBWQTWTG5uPypmb38XXbZZTQ2NtLT08Pu3bvp6emhsbGRyy67rNqpmeWS53XWKyLiOeC9ZCPtngy8o7xpmc0Mw8PDXHPNNbS2tlJfX09rayvXXHMNw8PD1U7NLJc8RWRWGufqg8AtZc7HbMbZvHnzhNtm01meYU++QjaGVV9E3Cfp1cAj5U3LbGaYM2cOq1atoq6ujk984hNcdtllrFq1ijlz5lQ7NbNccg178nLiYU9sOunp6eHjH/84u3btYmhoiPr6ehoaGrj88stpa2urdnpmLyo87ImZlU9bWxuXX345xx9/PAcddBDHH3+8C4jVFD+JmJnZpPwkYmZmUy7PHOufHSP8LLApIn459SmZmVmtyPMk0gJ8gmwI+HlkQ58sI/vo8PNlzM3MzKa5PEVkPnBiRHwuIj4HvIlsKPi/AD5a5KaS/qOkhyRtltQjqUHSYkn3SNoi6TpJB6djD0nbW9L+RSXXWZXiv5J0WpFczMysuDxF5JXA7pLtIeCYiPh/o+K5SJoHfBpoiYhmoA44C/g6cHFEvAZ4GmhPp7QDT6f4xek4JJ2Qzns92ZPROkl1+5uPmZkVl6eIfA+4R9JqSauB/wX8d0mHAw8XvO8ssqHlZwGHAb8B3k42gyLAVcCZaX152ibtf4ckpfi1EbE7IraSzc9+UsF8zMysgEmLSER8lawd5Jm0fCIivhIRz0fEh/f3hhGxDfgH4NdkxeNZsoEdn4mIkQGDBsnaX0h/H0/nDqfjjyqNj3HOPiStlNQvqX/Hjh37m7JZWfX09NDc3ExdXR3Nzc309PRUOyWz3PIMewJwP7Bt5HhJCyPi10VuKGk22VPEYrKidAPZ66iyiYj1wHrIvhMp573M9kdPTw+dnZ10d3ezdOlS+vr6aG/P3uT6g0OrBXmGgu8AngA2kg3AeCsHNhDjXwJbI2JHRAwB3yebw70xvd6CrDF/W1rfBixIucwCXgE8VRof4xyzmtDV1UV3d/c+o/h2d3fT1dVV7dTMcsnTJvIZ4LUR8fqIeENE/ElEvOEA7vlr4BRJh6W2jXeQta30Au9Px6wAbk7rG9I2af+P06RYG4CzUu+txcAS4N4DyMus4gYGBli6dOk+saVLlzIwMFCljMz2T54i8jhZO8SUiIh7yBrI7yebcvcgsldN5wGflbSFrM2jO53SDRyV4p8lm66XiHgIuJ6sAP0QODci9k5VnmaV0NTURF9f3z6xvr4+mpqaqpSR2f7J0ybyKHCXpFsp6dIbEd8oetOIWA2sHuM+L+ldFRG7gA+Mc50uwM/9VrM6Oztpb29/SZuIX2dZrchTRH6dloPTYmZTpK2tjZ/97Gecfvrp7N69m0MOOYRzzjnHjepWMyYtIhFx/uhYSQO4mR2Anp4ebr31Vm6//fZ9nkTe8pa3uJBYTRi3TURSX8n6d0ftdgO22RRw7yyrdRM1rB9est48ap/KkIvZjOPeWVbrJioiMc76WNtmVoB7Z1mtm6hto1HSe8gKTaOk96a4yD74M7MD5N5ZVusmKiI/Ad5dsv6ukn0/LVtGZjPISON5R0cHAwMDNDU10dXV5UZ1qxnjFpGI+FglEzEzs9rjrrpmVeQBGK3WKRuGauZoaWmJ/v7+aqdhBkBzczNr166ltbX1xVhvby8dHR1s3ry5ipmZ7UvSpohoGR2f6DuRD6S/i8uZmNlM5i6+Vusm6uK7Kv39p0okYjYTuYuv1bqJ2kSekvQjYLGkDaN3RsS7xzjHzPZDZ2cnH/rQhzj88MN57LHHOPbYY3n++edZs2ZNtVMzy2WiIvLXwInAd4H/Wpl0zGaubHods9oyURffPcDdkt4SETsk/VGK/9+KZWf2MtfV1cV11103ZsO6e2dZLZi0d5akZrKnkTlkX6vvAFZERE12HXHvLJtO6urq2LVrF/X19S/GhoaGaGhoYO9ez7Fm08d+984qsR74bEQcGxELgc+lmJkdoKamJs4//3yam5upq6ujubmZ888/3w3rVjPyFJHDI6J3ZCMi7mLfEX73m6RGSTdK+ldJA5LeLGmOpI2SHkl/Z6djJekSSVskPSDpxJLrrEjHPyJpxfh3NJueWltbueCCC3jyySeJCJ588kkuuOCCfV5vmU1neYrIo5L+XtKitHyJbCrbA7EG+GFEvA54IzBANnf6nRGxBLgzbQOcDixJy0rgUgBJc8im2D2ZbFrd1SOFx6xW3HTTTRxxxBEceuihABx66KEcccQR3HTTTVXOzCyfPEXkbGAu8H2yb0aOTrFCJL0C+AugG7IG/Ih4BlgOXJUOuwo4M60vB66OzN1kIwq/CjgN2BgROyPiaWAjsKxoXmbVMDg4yA033MDWrVt54YUX2Lp1KzfccAODg4PVTs0slzzT4z4NfHoK77mYrHH+O5LeCGwCPgMcExG/Scf8Fjgmrc8DHi85fzDFxou/hKSVZE8xLFy4cGp+hZmZ5XoSmWqzyL4/uTQi/gx4nj+8ugIgsi5jUzaoV0Ssj4iWiGiZO3fuVF3W7IDNnz+fFStW0Nvby9DQEL29vaxYsYL58+dXOzWzXKpRRAaBwYi4J23fSFZUnkivqUh/t6f924AFJefPT7Hx4mY146KLLmJ4eJizzz6bhoYGzj77bIaHh7nooouqnZpZLhUvIhHxW+BxSa9NoXcADwMbgJEeViuAm9P6BuAjqZfWKcCz6bXXHcCpkmanBvVTU8ysZrS1tbFmzRoOPzzr8Hj44YezZs0af2hoNSPPx4ZzgXOARZS0oUTEgTSu/ynwbeBgsp5eHyMraNcDC4HHgA9GxE5lY0H8I1mj+e+Bj0VEf7rO2cAX02W7IuI7k93bHxuame2/8T42zFNEfgb8T7IG8Bc/oY2Imhzd10XEzGz/jVdE8sxseFhEnFeGnMzMrMblaRO5RdI7y56JmZnVnHGfRCT9jqybrYAvStoNDKXtiIgjK5OimZlNVxMNBX9EJRMxM7PaM+nrLEl35omZmdnMM9HrrAay0XqPTt9hjEy7diTjDC9iZmYzy0S9sz4O/C3w74D7S+LPkX23YWZmM9xEbSJrgDWSOiJibQVzMjOzGpHnO5Ftkt47KvYs8GBEbB/rBDMzmxnyFJF24M3AyOyGbyP7en2xpK9ExHfLlJuZmU1zeT42rAeaIuJ9EfE+4ASy70dOBvwlu9kB6ujooKGhAUk0NDTQ0dFR7ZTMcstTROZHxBMl29uBBRGxk+zjQzMrqKOjg3Xr1tHY2IgkGhsbWbdunQuJ1Yw8AzCuIxtZ94YUeh/ZnCD/CbglIlrLmuEU8wCMNp3U19dTV1fHCy+8wNDQEPX19Rx00EHs3buXoSH/fzSbPsYbgDHPk8i5wJXAn6blauDciHi+1gqI2XQzPDzMnj17uPDCC3n++ee58MIL2bNnD8PDw9VOzSyXSZ9EXm78JGLTiSTOOOMMfvCDH7wYe9e73sUtt9zCTPu3adNb4aHgU/ferwOvJPtq3QMwmk2hW265hVmzZrF3717q6urYu3fv5CeZTRN5XmddBLw7Il4REUdGxBFTUUAk1Un6haRb0vZiSfdI2iLpOkkHp/ghaXtL2r+o5BqrUvxXkk470JzMKi2buJMXnzpG/o7Ezaa7PEXkiYgYKMO9PwOUXvfrwMUR8RrgabLvU0h/n07xi9NxSDoBOAt4PdnUuesk1ZUhT7OyOeig7J/gCy+8sM/fkbjZdJfnf6n96UmgTdJ7R5YDuamk+cBfk82zTppH/e3AjemQq4Az0/rytE3a/450/HLg2ojYHRFbgS3ASQeSl1mljffqyq+0rFbk+WL9SOD3wKklsQC+fwD3/W/A54GROUuOAp6JiJEuKYP8YaTgecDjABExLOnZdPw84O6Sa5aesw9JK4GVAAsXLjyAtM3MrNSkRSQiPjaVN5R0BrA9IjZJettUXns8EbEeWA9Z76xK3NPMbCbIMynV8ZLulLQ5bb9B0pcO4J5vBd4t6d+Aa8leY60BGiWNFLX5wLa0vg1YkO49C3gF8FRpfIxzzMysAvK0iXwLWEUa4iQiHiBr0C4kIlZFxPyIWJSu8+OI+DDZAI/vT4etAG5O6xvSNmn/jyPrwrIBOCv13loMLAHuLZqXmZntvzxtIodFxL2juhyW43Pa84BrJX0N+AXQneLdwHclbQF2kgpYRDwk6Xrg4ZTPuRHh1kgzswrKU0SelHQcWWM6kt4P/GYqbh4RdwF3pfVHGaN3VUTsAj4wzvldQNdU5GJmZvsvTxE5l6xR+nWStgFbgQ+XNSszM6sJk7aJRMSjEfGXwFzgdRGxFHhP2TMzM7NpL/dnsWnU3t+lzc+WKR8zM6shRcdW8MA+ZmZWuIj4gz0zMxu/YV3S7xi7WAg4tGwZmZlZzRi3iETEEePtMzMzg+Kvs8zMzFxEzMysOBcRMzMrzEXEzMwKcxExM7PCXETMzKwwFxEzMyvMRcTMzApzETEzs8IqXkQkLZDUK+lhSQ9J+kyKz5G0UdIj6e/sFJekSyRtkfSApBNLrrUiHf+IpBXj3dPMzMqjGk8iw8DnIuIE4BTgXEknAF8A7oyIJcCdaRvgdLL505cAK4FLISs6wGrgZLIZEVePFB4zM6uMiheRiPhNRNyf1n8HDADzgOXAVemwq4Az0/py4OrI3A00SnoVcBqwMSJ2RsTTwEZgWQV/ipnZjFfVNhFJi4A/A+4BjomIkbnbfwsck9bnAY+XnDaYYuPFzcysQqpWRCT9EfBPwN9GxHOl+yIimMI5SyStlNQvqX/Hjh1TdVkzsxmvKkVEUj1ZAfleRHw/hZ9Ir6lIf7en+DZgQcnp81NsvPhLRMT6iGiJiJa5c+dO3Q8xM5vhqtE7S0A3MBAR3yjZtQEY6WG1Ari5JP6R1EvrFODZ9NrrDuBUSbNTg/qpKWZmZhUy7qRUZfRW4G+AByX9MsW+CFwIXC+pHXgM+GDadxvwTmAL8HvgYwARsVPSV4H70nFfiYidlfkJZmYGoKz5YeZoaWmJ/v7+aqdhBkD2YD62mfZv06Y3SZsiomV03F+sm5lZYS4iZmZWmIuImZkV5iJiZmaFuYiYmVlhLiJmZlaYi4iZmRXmImJmZoW5iJiZWWEuImZmVpiLiJmZFeYiYmZmhbmImJlZYS4iZmZWmIuImZkV5iJiZmaFuYiYmVlhNV9EJC2T9CtJWyR9odr5mI2QNOlyoOdPdg2zcqvGHOtTRlId8E3gr4BB4D5JGyLi4epmZpZveltPj2u1rqaLCHASsCUiHgWQdC2wHHARsan15VeU5bKx+siK35MvP1ue69qMVOtFZB7weMn2IHDy6IMkrQRWAixcuLAymdnLis5/rtopTInZs2ez88vVzsJeTmq9iOQSEeuB9QAtLS1+R2D7za+WzMZW6w3r24AFJdvzU8zMzCqg1ovIfcASSYslHQycBWyock5mZjNGTb/OiohhSZ8C7gDqgCsi4qEqp2VmNmPUdBEBiIjbgNuqnYeZ2UxU66+zzMysilxEzMysMBcRMzMrzEXEzMwK00z7iErSDuCxaudhNoajgSernYTZOI6NiLmjgzOuiJhNV5L6I6Kl2nmY7Q+/zjIzs8JcRMzMrDAXEbPpY321EzDbX24TMTOzwvwkYmZmhbmImJlZYS4iZlUm6QpJ2yVtrnYuZvvLRcSs+q4EllU7CbMiXETMqiwifgrsrHYeZkW4iJiZWWEuImZmVpiLiJmZFeYiYmZmhbmImFWZpB7g58BrJQ1Kaq92TmZ5edgTMzMrzE8iZmZWmIuImZkV5iJiZmaFuYiYmVlhLiJmZlaYi4iZmRXmImJmZoX9f26S39kH4pf9AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Harsh filtering"
      ],
      "metadata": {
        "id": "rbSpL0FwuPYY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Remove elements with links"
      ],
      "metadata": {
        "id": "Rf1X5QqUmROp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "stop_chrt = ['http', 'url', 'www', 'html']\n",
        "junk_elems = [(idx, sent) for lines in [lines_eng, lines_rus]\n",
        "              for idx, sent in enumerate(lines) \n",
        "              if any(sw in sent for sw in stop_chrt)]\n",
        "junk_elems[:20], len(junk_elems)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_ikfff6wilkL",
        "outputId": "ad1780dc-5ace-43a7-8305-8451ec7ea571"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([(561,\n",
              "   'This can be consulted on the internet: www.cathedralerussenice.org/spip/spip.php?article61. In addition, following the judgement given by the Court of Appeal in Aix-en-Provence on May 19th 2011, the Religious Association ACOR decided to take the case to the final court of appeal.\\n'),\n",
              "  (648,\n",
              "   'To know exactly the times of the bus, follow this link: http://217.141.151.2/orari/urbani_minori/html/MAS0C.htm\\n'),\n",
              "  (793,\n",
              "   'There will be offered four thematic sessions depending on the child’s age. A. Brodeles st., Viesite, Jekabpils novads, LV-5237, Latvija +371 65245549, +371 29116334 viesitesmuzejsselija@inbox.lv www.viestie.lt\\n'),\n",
              "  (812,\n",
              "   'His carved sculpture of a woman was displayed in an exhibition together with M. K. Čiurlionis paintings.\\n'),\n",
              "  (960, 'www.koktelnieki.lv\\n'),\n",
              "  (1167, 'http://mp.zarabotak.com/Jeff%20Buckley.php\\n'),\n",
              "  (1168, 'http://mp.zarabotak.com/Catedral.php\\n'),\n",
              "  (1169, 'http://mp.zarabotak.com/Gary%20Burton.php\\n'),\n",
              "  (1170, 'http://mp.zarabotak.com/David%20Cassidy.php\\n'),\n",
              "  (1171, 'http://mp.zarabotak.com/U.D.O..php\\n'),\n",
              "  (1172, 'http://mp.zarabotak.com/Donnie%20McClurkin.php\\n'),\n",
              "  (1180, '- https://www.smallbusinessforums.org/sho ... ss-account\\n'),\n",
              "  (1224, 'http://www.fivefiles.com/files/2516/\\n'),\n",
              "  (1226, 'http://depositfiles.com/folders/8BVCI7LS0\\n'),\n",
              "  (1230,\n",
              "   'Exxon Valdez oil spill http://www.youtube.com/watch?v=dtF-4JvSh8o&sns=tw. Nature can to get rid of all ills, misfortune and razrusheny: http://www.youtube.com/watch?v=GUwnn99YOC0&sns=tw. Profit Pollution and Deception BP and the Oil Spill BBC Documentary http://www.youtube.com/watch?v=8zGFvzMMO9w&sns=tw. Dakota Access Pipeline Company Attacks Native American Protesters with Dogs & Pepper Spray http://www.youtube.com/watch?v=kuZcx2zEo4k&sns=tw. Environmental problems of the Earth! http://www.youtube.com/watch?v=1pe-KV6J-uE&sns=fb\\n'),\n",
              "  (1248, '- http://www.music-video.ru\\n'),\n",
              "  (1249, 'http://nnprog.ru/ - Фильмы, видиоклипы, MP3.\\n'),\n",
              "  (1250, 'http://marilynmanson.ru/archive/video/clips.html ЗДЕСЬ!\\n'),\n",
              "  (1264,\n",
              "   'Exxon Valdez oil spill http://www.youtube.com/watch?v=dtF-4JvSh8o&sns=tw. Nature can to get rid of all ills, misfortune and razrusheny: http://www.youtube.com/watch?v=GUwnn99YOC0&sns=tw. Profit Pollution and Deception BP and the Oil Spill BBC Documentary http://www.youtube.com/watch?v=8zGFvzMMO9w&sns=tw. Dakota Access Pipeline Company Attacks Native American Protesters with Dogs & Pepper Spray http://www.youtube.com/watch?v=kuZcx2zEo4k&sns=tw. Environmental problems of the Earth! http://www.youtube.com/watch?v=1pe-KV6J-uE&sns=fb\\n'),\n",
              "  (1281,\n",
              "   'Between us and other animals and plants, there is no difference, because we are all children of the Earth and we are all one family. We all the world should live according to the laws of mother nature, to save the Earth from ecological disaster.Exxon Valdez oil spill http://www.youtube.com/watch?v=dtF-4JvSh8o&sns=tw. Nature can to get rid of all ills, misfortune and razrusheny: http://www.youtube.com/watch?v=GUwnn99YOC0&sns=tw. Profit Pollution and Deception BP and the Oil Spill BBC Documentary http://www.youtube.com/watch?v=8zGFvzMMO9w&sns=tw. Dakota Access Pipeline Company Attacks Native American Protesters with Dogs & Pepper Spray http://www.youtube.com/watch?v=kuZcx2zEo4k&sns=tw. Environmental problems of the Earth! http://www.youtube.com/watch?v=1pe-KV6J-uE&sns=fb\\n')],\n",
              " 854333)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "filtered_lines = {}\n",
        "junk_idxs = set([x[0] for x in junk_elems])\n",
        "for data, key in [(lines_rus, 'rus'), (lines_eng, 'eng')]:\n",
        "    filtered_lines[key] = [sent for idx, sent in enumerate(data) if idx not in junk_idxs]"
      ],
      "metadata": {
        "id": "Z-BWGJ06u7zt"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stats_hard_filtered = get_statistics(filtered_lines['rus'], filtered_lines['eng'])\n",
        "print(stats_hard_filtered[0])\n",
        "plot_counts_boxplot(stats_hard_filtered)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "sX67ibpyv2iQ",
        "outputId": "276790de-54df-43fc-c6d6-0abccb5d2b99"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Lenght statistics:\n",
            "Rus:{'mean': 77.41984288225754, 'argmin': 675, 'argmax': 9230692, 'min': 2, 'max': 6775}\n",
            "Eng{'mean': 80.71968020958668, 'argmin': 675, 'argmax': 9230692, 'min': 2, 'max': 6775}\n",
            "Rus / Eng {'mean': 1.038811039612353, 'argmin': 8540394, 'argmax': 7066074, 'min': 0.018404907975460124, 'max': 56.0}\n",
            "Pairs\n",
            "Smallest: ('?\\n', '?\\n')\n",
            "Longest: ('\").appendTo(this.element).addClass(\"ui-slider-range ui-widget-header\"+(c.range===\"min\"c.range===\"max\"?\" ui-slider-range-\"+c.range:\"\"))}for(var i=e.length;i\"); this.handles=e.add(a(g.join(\"\")).appendTo(d.element));this.handle=this.handles.eq(0);this.handles.add(this.range).filter(\"a\").click(function(b){b.preventDefault()}).hover(function(){c.disableda(this).addClass(\"ui-state-hover\")},function(){a(this).removeClass(\"ui-state-hover\")}).focus(function(){if(c.disabled)a(this).blur();else{a(\".ui-slider .ui-state-focus\").removeClass(\"ui-state-focus\");a(this).addClass(\"ui-state-focus\")}}).blur(function(){a(this).removeClass(\"ui-state-focus\")});this.handles.each(function(b){a(this).data(\"index.ui-slider-handle\", b)});this.handles.keydown(function(b){var f=true,j=a(this).data(\"index.ui-slider-handle\"),l,o,n;if(!d.options.disabled){switch(b.keyCode){case a.ui.keyCode.HOME:case a.ui.keyCode.END:case a.ui.keyCode.PAGE_UP:case a.ui.keyCode.PAGE_DOWN:case a.ui.keyCode.UP:case a.ui.keyCode.RIGHT:case a.ui.keyCode.DOWN:case a.ui.keyCode.LEFT:f=false;if(!d._keySliding){d._keySliding=true;a(this).addClass(\"ui-state-active\");l=d._start(b,j);if(l===false)return}break}n=d.options.step;l=d.options.values&&d.options.values.length? (o=d.values(j)):(o=d.value());switch(b.keyCode){case a.ui.keyCode.HOME:o=d._valueMin();break;case a.ui.keyCode.END:o=d._valueMax();break;case a.ui.keyCode.PAGE_UP:o=d._trimAlignValue(l+(d._valueMax()-d._valueMin())/5);break;case a.ui.keyCode.PAGE_DOWN:o=d._trimAlignValue(l-(d._valueMax()-d._valueMin())/5);break;case a.ui.keyCode.UP:case a.ui.keyCode.RIGHT:if(l===d._valueMax())return;o=d._trimAlignValue(l+n);break;case a.ui.keyCode.DOWN:case a.ui.keyCode.LEFT:if(l===d._valueMin())return;o=d._trimAlignValue(l- n);break}d._slide(b,j,o);return f}}).keyup(function(b){var f=a(this).data(\"index.ui-slider-handle\");if(d._keySliding){d._keySliding=false;d._stop(b,f);d._change(b,f);a(this).removeClass(\"ui-state-active\")}});this._refreshValue();this._animateOff=false},destroy:function(){this.handles.remove();this.range.remove();this.element.removeClass(\"ui-slider ui-slider-horizontal ui-slider-vertical ui-slider-disabled ui-widget ui-widget-content ui-corner-all\").removeData(\"slider\").unbind(\".slider\");this._mouseDestroy(); return this},_mouseCapture:function(d){var c=this.options,e,h,g,i,b;if(c.disabled)return false;this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()};this.elementOffset=this.element.offset();e=this._normValueFromMouse({x:d.pageX,y:d.pageY});h=this._valueMax()-this._valueMin()+1;i=this;this.handles.each(function(f){var j=Math.abs(e-i.values(f));if(h>j){h=j;g=a(this);b=f}});if(c.range===true&&this.values(1)===c.min){b+=1;g=a(this.handles[b])}if(this._start(d,b)===false)return false; this._mouseSliding=true;i._handleIndex=b;g.addClass(\"ui-state-active\").focus();c=g.offset();this._clickOffset=!a(d.target).parents().andSelf().is(\".ui-slider-handle\")?{left:0,top:0}:{left:d.pageX-c.left-g.width()/2,top:d.pageY-c.top-g.height()/2-(parseInt(g.css(\"borderTopWidth\"),10)0)-(parseInt(g.css(\"borderBottomWidth\"),10)0)+(parseInt(g.css(\"marginTop\"),10)0)};this.handles.hasClass(\"ui-state-hover\")this._slide(d,b,e);return this._animateOff=true},_mouseStart:function(){return true},_mouseDrag:function(d){var c= this._normValueFromMouse({x:d.pageX,y:d.pageY});this._slide(d,this._handleIndex,c);return false},_mouseStop:function(d){this.handles.removeClass(\"ui-state-active\");this._mouseSliding=false;this._stop(d,this._handleIndex);this._change(d,this._handleIndex);this._clickOffset=this._handleIndex=null;return this._animateOff=false},_detectOrientation:function(){this.orientation=this.options.orientation===\"vertical\"?\"vertical\":\"horizontal\"},_normValueFromMouse:function(d){var c;if(this.orientation===\"horizontal\"){c= this.elementSize.width;d=d.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)}else{c=this.elementSize.height;d=d.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)}c=d/c;if(c>1)c=1;if(c<0)c=0;if(this.orientation===\"vertical\")c=1-c;d=this._valueMax()-this._valueMin();return this._trimAlignValue(this._valueMin()+c*d)},_start:function(d,c){var e={handle:this.handles[c],value:this.value()};if(this.options.values&&this.options.values.length){e.value=this.values(c); e.values=this.values()}return this._trigger(\"start\",d,e)},_slide:function(d,c,e){var h;if(this.options.values&&this.options.values.length){h=this.values(c?0:1);if(this.options.values.length===2&&this.options.range===true&&(c===0&&e>hc===1&&e1){this.options.values[d]=this._trimAlignValue(c);this._refreshValue();this._change(null,d)}else if(arguments.length)if(a.isArray(arguments[0])){e=this.options.values;h=arguments[0];for(g=0;g=this._valueMax())return this._valueMax();var c=this.options.step>0?this.options.step:1,e=(d-this._valueMin())%c;d=d-e;if(Math.abs(e)*2>=c)d+=e>0?c:-c;return parseFloat(d.toFixed(5))},_valueMin:function(){return this.options.min},_valueMax:function(){return this.options.max},_refreshValue:function(){var d= this.options.range,c=this.options,e=this,h=!this._animateOff?c.animate:false,g,i={},b,f,j,l;if(this.options.values&&this.options.values.length)this.handles.each(function(o){g=(e.values(o)-e._valueMin())/(e._valueMax()-e._valueMin())*100;i[e.orientation===\"horizontal\"?\"left\":\"bottom\"]=g+\"%\";a(this).stop(1,1)[h?\"animate\":\"css\"](i,c.animate);if(e.options.range===true)if(e.orientation===\"horizontal\"){if(o===0)e.range.stop(1,1)[h?\"animate\":\"css\"]({left:g+\"%\"},c.animate);if(o===1)e.range[h?\"animate\":\"css\"]({width:g- b+\"%\"},{queue:false,duration:c.animate})}else{if(o===0)e.range.stop(1,1)[h?\"animate\":\"css\"]({bottom:g+\"%\"},c.animate);if(o===1)e.range[h?\"animate\":\"css\"]({height:g-b+\"%\"},{queue:false,duration:c.animate})}b=g});else{f=this.value();j=this._valueMin();l=this._valueMax();g=l!==j?(f-j)/(l-j)*100:0;i[e.orientation===\"horizontal\"?\"left\":\"bottom\"]=g+\"%\";this.handle.stop(1,1)[h?\"animate\":\"css\"](i,c.animate);if(d===\"min\"&&this.orientation===\"horizontal\")this.range.stop(1,1)[h?\"animate\":\"css\"]({width:g+\"%\"}, c.animate);if(d===\"max\"&&this.orientation===\"horizontal\")this.range[h?\"animate\":\"css\"]({width:100-g+\"%\"},{queue:false,duration:c.animate});if(d===\"min\"&&this.orientation===\"vertical\")this.range.stop(1,1)[h?\"animate\":\"css\"]({height:g+\"%\"},c.animate);if(d===\"max\"&&this.orientation===\"vertical\")this.range[h?\"animate\":\"css\"]({height:100-g+\"%\"},{queue:false,duration:c.animate})}}});a.extend(a.ui.slider,{version:\"1.8.16\"})})(jQuery); (function(a,d){function c(){return++h}function e(){return++g}var h=0,g=0;a.widget(\"ui.tabs\",{options:{add:null,ajaxOptions:null,cache:false,cookie:null,collapsible:false,disable:null,disabled:[],enable:null,event:\"click\",fx:null,idPrefix:\"ui-tabs-\",load:null,panelTemplate:\"\\n', '\").appendTo(this.element).addClass(\"ui-slider-range ui-widget-header\"+(c.range===\"min\"c.range===\"max\"?\" ui-slider-range-\"+c.range:\"\"))}for(var i=e.length;i\"); this.handles=e.add(a(g.join(\"\")).appendTo(d.element));this.handle=this.handles.eq(0);this.handles.add(this.range).filter(\"a\").click(function(b){b.preventDefault()}).hover(function(){c.disableda(this).addClass(\"ui-state-hover\")},function(){a(this).removeClass(\"ui-state-hover\")}).focus(function(){if(c.disabled)a(this).blur();else{a(\".ui-slider .ui-state-focus\").removeClass(\"ui-state-focus\");a(this).addClass(\"ui-state-focus\")}}).blur(function(){a(this).removeClass(\"ui-state-focus\")});this.handles.each(function(b){a(this).data(\"index.ui-slider-handle\", b)});this.handles.keydown(function(b){var f=true,j=a(this).data(\"index.ui-slider-handle\"),l,o,n;if(!d.options.disabled){switch(b.keyCode){case a.ui.keyCode.HOME:case a.ui.keyCode.END:case a.ui.keyCode.PAGE_UP:case a.ui.keyCode.PAGE_DOWN:case a.ui.keyCode.UP:case a.ui.keyCode.RIGHT:case a.ui.keyCode.DOWN:case a.ui.keyCode.LEFT:f=false;if(!d._keySliding){d._keySliding=true;a(this).addClass(\"ui-state-active\");l=d._start(b,j);if(l===false)return}break}n=d.options.step;l=d.options.values&&d.options.values.length? (o=d.values(j)):(o=d.value());switch(b.keyCode){case a.ui.keyCode.HOME:o=d._valueMin();break;case a.ui.keyCode.END:o=d._valueMax();break;case a.ui.keyCode.PAGE_UP:o=d._trimAlignValue(l+(d._valueMax()-d._valueMin())/5);break;case a.ui.keyCode.PAGE_DOWN:o=d._trimAlignValue(l-(d._valueMax()-d._valueMin())/5);break;case a.ui.keyCode.UP:case a.ui.keyCode.RIGHT:if(l===d._valueMax())return;o=d._trimAlignValue(l+n);break;case a.ui.keyCode.DOWN:case a.ui.keyCode.LEFT:if(l===d._valueMin())return;o=d._trimAlignValue(l- n);break}d._slide(b,j,o);return f}}).keyup(function(b){var f=a(this).data(\"index.ui-slider-handle\");if(d._keySliding){d._keySliding=false;d._stop(b,f);d._change(b,f);a(this).removeClass(\"ui-state-active\")}});this._refreshValue();this._animateOff=false},destroy:function(){this.handles.remove();this.range.remove();this.element.removeClass(\"ui-slider ui-slider-horizontal ui-slider-vertical ui-slider-disabled ui-widget ui-widget-content ui-corner-all\").removeData(\"slider\").unbind(\".slider\");this._mouseDestroy(); return this},_mouseCapture:function(d){var c=this.options,e,h,g,i,b;if(c.disabled)return false;this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()};this.elementOffset=this.element.offset();e=this._normValueFromMouse({x:d.pageX,y:d.pageY});h=this._valueMax()-this._valueMin()+1;i=this;this.handles.each(function(f){var j=Math.abs(e-i.values(f));if(h>j){h=j;g=a(this);b=f}});if(c.range===true&&this.values(1)===c.min){b+=1;g=a(this.handles[b])}if(this._start(d,b)===false)return false; this._mouseSliding=true;i._handleIndex=b;g.addClass(\"ui-state-active\").focus();c=g.offset();this._clickOffset=!a(d.target).parents().andSelf().is(\".ui-slider-handle\")?{left:0,top:0}:{left:d.pageX-c.left-g.width()/2,top:d.pageY-c.top-g.height()/2-(parseInt(g.css(\"borderTopWidth\"),10)0)-(parseInt(g.css(\"borderBottomWidth\"),10)0)+(parseInt(g.css(\"marginTop\"),10)0)};this.handles.hasClass(\"ui-state-hover\")this._slide(d,b,e);return this._animateOff=true},_mouseStart:function(){return true},_mouseDrag:function(d){var c= this._normValueFromMouse({x:d.pageX,y:d.pageY});this._slide(d,this._handleIndex,c);return false},_mouseStop:function(d){this.handles.removeClass(\"ui-state-active\");this._mouseSliding=false;this._stop(d,this._handleIndex);this._change(d,this._handleIndex);this._clickOffset=this._handleIndex=null;return this._animateOff=false},_detectOrientation:function(){this.orientation=this.options.orientation===\"vertical\"?\"vertical\":\"horizontal\"},_normValueFromMouse:function(d){var c;if(this.orientation===\"horizontal\"){c= this.elementSize.width;d=d.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)}else{c=this.elementSize.height;d=d.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)}c=d/c;if(c>1)c=1;if(c<0)c=0;if(this.orientation===\"vertical\")c=1-c;d=this._valueMax()-this._valueMin();return this._trimAlignValue(this._valueMin()+c*d)},_start:function(d,c){var e={handle:this.handles[c],value:this.value()};if(this.options.values&&this.options.values.length){e.value=this.values(c); e.values=this.values()}return this._trigger(\"start\",d,e)},_slide:function(d,c,e){var h;if(this.options.values&&this.options.values.length){h=this.values(c?0:1);if(this.options.values.length===2&&this.options.range===true&&(c===0&&e>hc===1&&e1){this.options.values[d]=this._trimAlignValue(c);this._refreshValue();this._change(null,d)}else if(arguments.length)if(a.isArray(arguments[0])){e=this.options.values;h=arguments[0];for(g=0;g=this._valueMax())return this._valueMax();var c=this.options.step>0?this.options.step:1,e=(d-this._valueMin())%c;d=d-e;if(Math.abs(e)*2>=c)d+=e>0?c:-c;return parseFloat(d.toFixed(5))},_valueMin:function(){return this.options.min},_valueMax:function(){return this.options.max},_refreshValue:function(){var d= this.options.range,c=this.options,e=this,h=!this._animateOff?c.animate:false,g,i={},b,f,j,l;if(this.options.values&&this.options.values.length)this.handles.each(function(o){g=(e.values(o)-e._valueMin())/(e._valueMax()-e._valueMin())*100;i[e.orientation===\"horizontal\"?\"left\":\"bottom\"]=g+\"%\";a(this).stop(1,1)[h?\"animate\":\"css\"](i,c.animate);if(e.options.range===true)if(e.orientation===\"horizontal\"){if(o===0)e.range.stop(1,1)[h?\"animate\":\"css\"]({left:g+\"%\"},c.animate);if(o===1)e.range[h?\"animate\":\"css\"]({width:g- b+\"%\"},{queue:false,duration:c.animate})}else{if(o===0)e.range.stop(1,1)[h?\"animate\":\"css\"]({bottom:g+\"%\"},c.animate);if(o===1)e.range[h?\"animate\":\"css\"]({height:g-b+\"%\"},{queue:false,duration:c.animate})}b=g});else{f=this.value();j=this._valueMin();l=this._valueMax();g=l!==j?(f-j)/(l-j)*100:0;i[e.orientation===\"horizontal\"?\"left\":\"bottom\"]=g+\"%\";this.handle.stop(1,1)[h?\"animate\":\"css\"](i,c.animate);if(d===\"min\"&&this.orientation===\"horizontal\")this.range.stop(1,1)[h?\"animate\":\"css\"]({width:g+\"%\"}, c.animate);if(d===\"max\"&&this.orientation===\"horizontal\")this.range[h?\"animate\":\"css\"]({width:100-g+\"%\"},{queue:false,duration:c.animate});if(d===\"min\"&&this.orientation===\"vertical\")this.range.stop(1,1)[h?\"animate\":\"css\"]({height:g+\"%\"},c.animate);if(d===\"max\"&&this.orientation===\"vertical\")this.range[h?\"animate\":\"css\"]({height:100-g+\"%\"},{queue:false,duration:c.animate})}}});a.extend(a.ui.slider,{version:\"1.8.16\"})})(jQuery); (function(a,d){function c(){return++h}function e(){return++g}var h=0,g=0;a.widget(\"ui.tabs\",{options:{add:null,ajaxOptions:null,cache:false,cookie:null,collapsible:false,disable:null,disabled:[],enable:null,event:\"click\",fx:null,idPrefix:\"ui-tabs-\",load:null,panelTemplate:\"\\n')\n",
            "Smallest ratio: ('123855(20.11.2014), D - KAY(21.11.2014), DisPaster(20.11.2014), fidlar(20.11.2014), Fyfnjkbq(25.11.2014), Gello(24.11.2014), gism666(11.02.2016), lycanthropunk(20.11.2014), mr.d4rky(20.11.2014), outspoken(13.12.2014), s1ndrom(22.11.2014), Volence(08.05.2016), Wasteland(20.11.2014), Wolftribes(09.01.2015), yxfxmx(10.11.2015)\\n', 'Z (1)\\n')\n",
            "Biggest ratio: ('-\\n', 'Cамые свежие новости о главных событиях власти,политики,спорта,культуры,происшествия,экономики,бизнеса,иносми -\\n')\n",
            "Counts of pairs: 11622112\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAD5CAYAAADWfRn1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcD0lEQVR4nO3df5RfdX3n8eeLyYQxEzCJiSlmEohu1GHnWIuzxB85PU6pgLQV/FEgYok4u1mRpnHZrgTHblwwu9bd2ka6shuaaPDgCIqWLHLEFAfdOVssE3Q1Mu1Jyg8zs0AikwAGBiaT9/5xPxO+CZm5d5J8f01ej3Pu+d77+d4f72/OSd65n5+KCMzMzCZySrUDMDOz2udkYWZmuZwszMwsl5OFmZnlcrIwM7NcThZmZpZrWrluLOlNwO0lRa8H/iNwayo/C3gMuDQi9koSsB64CHge+GhEPJTutQL4TLrP5yJi80TPnjt3bpx11lkn7LeYmZ0Mtm3b9quImHe071SJcRaSGoBBYClwDTAUEZ+XtAaYHRHXSboIWEWWLJYC6yNiqaQ5QB/QDgSwDXhbROwd73nt7e3R19dX3h9lZjbFSNoWEe1H+65S1VDnAf8cEY8DFwNjbwabgUvS/sXArZF5AJgl6QzgAmBrRAylBLEVuLBCcZuZGZVLFpcD3Wl/fkQ8kfafBOan/QXArpJrBlLZeOVmZlYhZU8WkqYD7wO+eeR3kdWBnZB6MEkrJfVJ6tuzZ8+JuKWZmSWVeLN4L/BQRDyVjp9K1Uukz92pfBBYWHJdSyobr/wwEbEhItojon3evKO2z5iZ2TGqRLJYzstVUABbgBVpfwVwV0n5lcq8HXgmVVfdC5wvabak2cD5qczMzCqkrMlCUjPwHuDbJcWfB94jaQfwu+kY4B7gEWAncAvwCYCIGAJuBB5M2w2pzKyudHd309bWRkNDA21tbXR3d+dfZFYjyjbOAiAi9gOvOaLsabLeUUeeG2Tdao92n03ApnLEaFYJ3d3ddHV1sXHjRpYtW0Zvby+dnZ0ALF++vMrRmeWryDiLSvM4C6s1bW1t3HTTTXR0dBwq6+npYdWqVWzfvr2KkZm9bKJxFk4WZhXQ0NDA8PAwjY2Nh8pGRkZoampidHS0ipGZvawWBuWZndRaW1vp7e09rKy3t5fW1tYqRWQ2OU4WZhXQ1dVFZ2cnPT09jIyM0NPTQ2dnJ11dXdUOzayQsjZwm1lmrBF71apV9Pf309rayrp169y4bXXDbRZmZga4zcLMzI6Tk4WZmeVysjAzs1xOFmZmlsvJwszMcjlZmJlZLicLMzPL5WRhZma5nCzMzCyXk4WZmeVysjAzs1xOFmZmlsvJwszMcjlZmJlZrrImC0mzJH1L0j9K6pf0DklzJG2VtCN9zk7nStKXJO2U9DNJ55TcZ0U6f4ekFeWM2czMXqncbxbrge9FxJuB3wT6gTXAfRGxBLgvHQO8F1iStpXAzQCS5gBrgaXAucDasQRjZmaVUbZkIenVwG8DGwEi4qWI2AdcDGxOp20GLkn7FwO3RuYBYJakM4ALgK0RMRQRe4GtwIXlitvMzF6pnG8Wi4E9wFck/UTS30hqBuZHxBPpnCeB+Wl/AbCr5PqBVDZeuZmZVUg5k8U04Bzg5oj4LWA/L1c5ARDZmq4nZF1XSSsl9Unq27Nnz4m4pZmZJeVMFgPAQET8OB1/iyx5PJWql0ifu9P3g8DCkutbUtl45YeJiA0R0R4R7fPmzTuhP8TM7GRXtmQREU8CuyS9KRWdBzwMbAHGejStAO5K+1uAK1OvqLcDz6TqqnuB8yXNTg3b56cyMzOrkGllvv8q4DZJ04FHgKvIEtQdkjqBx4FL07n3ABcBO4Hn07lExJCkG4EH03k3RMRQmeM2M7MSypoNppb29vbo6+urdhhmZnVF0raIaD/adx7BbWZmuZwszMwsl5OFmZnlcrIwM7NcThZmZpbLycLMzHI5WZiZWS4nCzMzy+VkYWZmuZwszMwsl5OFmZnlcrIwM7NcThZmZpbLycLMzHLlJgtJMyT9maRb0vESSb9f/tDMzKxWFHmz+ArwIvCOdDwIfK5sEZmZWc0pkizeEBFfAEYAIuJ5QGWNyszMakqRZPGSpFcBASDpDWRvGmZmdpIosgb3WuB7wEJJtwHvAj5azqDMzKy25CaLiNgq6SHg7WTVT6sj4ldlj8zMzGpGkd5Q7wcORMR3I+Ju4ICkS8ofmpmZ1YoibRZrI+KZsYOI2EdWNZVL0mOSfi7pp5L6UtkcSVsl7Uifs1O5JH1J0k5JP5N0Tsl9VqTzd0haMbmfaGZmx6tIsjjaOUXaOsZ0RMRbI6I9Ha8B7ouIJcB96RjgvcCStK0EboYsuZAlp6XAucDasQRjZmaVUSRZ9En6oqQ3pO2LwLbjeObFwOa0vxm4pKT81sg8AMySdAZwAbA1IoYiYi+wFbjwOJ5vZmaTVCRZrAJeAm5P24vANQXvH8D3JW2TtDKVzY+IJ9L+k8D8tL8A2FVy7UAqG6/8MJJWSuqT1Ldnz56C4ZmZWRFFekPt5+WqoslaFhGDkl4LbJX0j0fcOyTFMd77MBGxAdgA0N7efkLuaWZmmdxkIemNwJ8CZ5WeHxG/k3dtRAymz92SvkPW5vCUpDMi4olUzbQ7nT4ILCy5vCWVDQLvPqL8/rxnm5nZiVOkGuqbwE+AzwD/oWSbkKRmSaeN7QPnA9uBLcBYj6YVwF1pfwtwZeoV9XbgmVRddS9wvqTZqWH7/FRmZmYVUqRX04GIuPkY7j0f+I6ksed8PSK+J+lB4A5JncDjwKXp/HuAi4CdwPPAVQARMSTpRuDBdN4NETF0DPGYmdkxUsTE1fuSPktWVfQdSuaEquV/sNvb26Ovr6/aYZiZ1RVJ20qGORymyJvFWJVRadVTAK8/3sDMzKw+FOkNtbgSgZiZWe0qulLeZyRtSMdeKc/M7CRTdKW8l4B3pmOvlGdmdpLxSnlmZpbLK+WZmVmuIr2hPssrV8q7qpxBmZlZbSnSG+r7krbhlfLMzE5aRXpD3RcRT4+tlBcRv5J0XyWCMzOz2jBuspDUlBYempvmZZqTtrM4yhThZjax7u5u2traaGhooK2tje7u7mqHZFbYRNVQ/xb4JPA6ssWOxnpAPQv8dZnjMptSuru76erqYuPGjSxbtoze3l46OzsBWL58eZWjM8tXZG6oVRFxU4XiOSE8N5TVmra2Nm666SY6OjoOlfX09LBq1Sq2b99excjMXjbR3FC5ySLd4J28cj2LW09UgCeak4XVmoaGBoaHh2lsbDxUNjIyQlNTE6Ojo1WMzOxlEyWLIg3cXwP+G7AM+FdpO+rNzOzoWltb6e3tPayst7eX1tbWKkVkNjlFxlm0A2dHkVcQMzuqrq4uOjs7X9FmsW7dumqHZlZIkWSxHfgN4Ikyx2I2ZY01Yq9atYr+/n5aW1tZt26dG7etbhRp4O4B3gr8A4cvfvS+8oZ27NxmYWY2ecfVZkE23cclwH8G/qJkM7NJ8DgLq2dFpvv4oaQzgSUR8XeSZgAN5Q/NbOrwOAurd0V6Q/0b4FvA/0xFC4C/LWdQZlPNunXr2LhxIx0dHTQ2NtLR0cHGjRvdwG11o0g11DVkM80+CxARO4DXFn2ApAZJP5F0dzpeLOnHknZKul3S9FR+ajremb4/q+Qe16fyf5J0QfGfZ1Yb+vv7WbZs2WFly5Yto7+/v0oRmU1OkWTxYkS8NHYgaRppbYuCVgOlfyP+HPjLiPgXwF6gM5V3AntT+V+m85B0NnA58C+BC4EvS3I1mNUVj7OwelckWfxQ0qeBV0l6D/BN4H8VubmkFuD3gL9JxwJ+h6xaC2AzWeM5wMXpmPT9een8i4FvRMSLEfEosBM4t8jzzWrF2DiLnp4eRkZG6OnpobOzk66urmqHZlZIkXEWa8j+1/9zsskF7yH941/AXwGfAk5Lx68B9kXEgXQ8wMsz2C4AdgFExAFJz6TzFwAPlNyz9JpDJK0EVgIsWrSoYHhmleFxFlbvivSGOgjcAtySpixvKTKaW9LvA7sjYpukdx93pDkiYgOwAbJxFuV+ntlkLV++3MnB6lZuspB0P/C+dO42YLek/xMR/y7n0ncB75N0EdAEnA6sB2ZJmpbeLlqAwXT+ILAQGEjtIq8Gni4pH1N6jZmZVUCRNotXR8SzwAeAWyNiKXBe3kURcX1EtETEWWQN1D+IiCuAHuBD6bQVwF1pf0s6Jn3/g/QGswW4PPWWWgwsIRtNbmZmFVIkWUyTdAZwKXD3CXjmdcC1knaStUlsTOUbgdek8mvJ2kqIiF8AdwAPA98DrokIz+lsZlZBRRq4bwDuBXoj4kFJrwd2TOYhEXE/cH/af4Sj9GaKiGHgD8e5fh3g0UtmZlWS+2YREd+MiLdExCfS8SMR8cHyh2Y2tXhuKKtnRd4szOw4eW4oq3eFllWtN56i3GqN1+C2enDca3DXGycLqzVeg9vqwUTJosg4i2uPUvwMsC0ifnq8wZmdDMbmhip9s/DcUFZPinSdbQc+TjbFxgKyKT8uJBvR/akyxmY2ZXhuKKt3RRq4W4BzIuLXAJLWAt8FfptsRPcXyhee2dTguaGs3hVJFq+lZO1tYASYHxEvSHpxnGvM7AieG8rqWZFkcRvwY0lj03L8AfB1Sc1ko6rNzGyKKzLr7I2Svge8MxV9PCLGuhpdUbbIzMysZhRp4AZ4iGzRo++QzTrrBSPMJskjuK2eFek6uwpYCzwFjAIiW1b1LeUNzWzq6O7uZvXq1TQ3NwOwf/9+Vq9eDXgEt9WH3EF5aRbYpRHxdGVCOn4elGe1ZuHChYyOjnLbbbcdmu7jiiuuoKGhgV27dlU7PDNg4kF5RaqhdpENwjOzYzQwMMDmzZvp6OigsbGRjo4ONm/ezMDAQLVDMyukSG+oR4D7JX2Xki60EfHFskVlZmY1pcibxS+BrcB04LSSzcwKamlp4corrzxsBPeVV15JS0tLtUMzK6RI19n/dGRZWiPbzAr6whe+wOrVq/nYxz7G448/zplnnsno6Chf/KJf0K0+jPtmIam3ZP9rR3ztNbDNJmH58uWsX7+e5uZmJNHc3Mz69evdE8rqxkRvCM0l+21HfKcyxGI2pXm6D6tnE7VZxDj7Rzs2sxwelGf1bKI3i1mS3k+WUGZJ+kAqF/DqvBtLagJ+BJyanvOtiFgraTHwDeA1ZLPW/lFEvCTpVOBW4G3A08BlEfFYutf1QCfZoMA/iYh7J/1LzarIy6pavRt3UJ6kr0x0YURcNeGNJQHNEfFrSY1AL7AauBb4dkR8Q9L/AP5vRNws6RPAWyLi45IuB94fEZdJOhvoBs4FXgf8HfDGiBh3eTEPyrNa42VVrR5UfVlVSTPIksXVZGth/EZEHJD0DuCzEXGBpHvT/t+n3lZPAvOANQAR8V/SvQ6dN97znCys1nhZVasHxzuC+3ge3CDpp8BusrEa/wzsi4gD6ZQBstX3SJ+7ANL3z5BVVR0qP8o1pc9aKalPUt+ePXvK8XPMjtnYsqqlvKyq1ZOyjpdIVUVvlTSLbMbaN5fxWRuADZC9WZTrOWbHoquri8suu4zm5mZ++ctfsmjRIvbv38/69eurHZpZIRONs/jD9Ln4eB8SEfuAHuAdZI3lY0mqBRhM+4PAwvTMaWSN6E+Xlh/lGrO6U4mqX7MTbaJqqOvT553HcmNJ89IbBZJeBbwH6CdLGh9Kp60Axlbg25KOSd//ILK/VVuAyyWdmhLXEjwo0OrMunXruP3223n00Uc5ePAgjz76KLfffjvr1q2rdmhmhUxUDfW0pO8DiyVtOfLLiHhfzr3PADZLaiBLSndExN2SHga+IelzwE+Ajen8jcDX0pToQ8Dl6Tm/kHQH2RKuB4BrJuoJZVaL+vv7GRgYoK2tjf7+flpbW7nuuuvo7++vdmhmhUzUdXY6cA7wNeBfH/l9RPywvKEdO/eGslqzcOFCDhw4wNe//vVD4yw+/OEPM23aNK9nYTVjot5Q475ZRMRLwAOS3hkReyTNTOW/LlOcZlPa8PDwYRMJDg8PM3PmzGqHZVZIka6z8yX9BPgF8LCkbZKOnCvKzCYwODjI6Ogog4ODRMRhx2b1oEiy2ABcGxFnRsQi4N+nMjMrqKGhgYMHD7JgwQJOOeUUFixYwMGDB2loaKh2aGaFFBln0RwRPWMHEXG/pOaJLjCzwx04cIDR0VGGh4eBrErq+eefdzdaqxtF3iwekfRnks5K22fIllo1s0mYMWMGTU1NRARNTU3MmDGj2iGZFVYkWXyMbI6mb5ONuZibysxsEqZPn86mTZt48cUX2bRpE9OnT692SGaFFVlWdS/wJxWIxWxKGx4e5oILLmBkZITGxkamTfPqxFY/yjqRoJll5syZwwsvvMDBgwcBOHjwIC+88AJz5sypcmRmxThZmFWIJObOnXvYp1m9cLIwq4ChoSHWrFlzWLJYs2YNQ0ND1Q7NrJDcZJEmBPy0pA2SNo1tlQjObCrp6Ohg+/btjI6Osn379sNWzTOrdUVa2O4C/jfZcqaewM/sGLS0tHDppZcya9asQ+tZ7Nu3j5aWlmqHZlZIkWQxIyKuK3skZlPYJZdcwpe//GVOPfVUIoIXXniBZ599lo985CPVDs2skCJtFndLuqjskZhNYT09PVx//fWHtVlcf/319PT05F9sVgMmmqL8OSAAAc3Ai8BIOo6IOL1SQU6Wpyi3WtPQ0MDw8DCNjY2HykZGRmhqamJ01LW7VhsmmqJ83DeLiDgtIk5Pn6dExKtKjms2UZjVotbWVnp7ew8r6+3tpbW1tUoRmU1ObpuFpPsi4ry8MjMbX1dXF5dddhnNzc2H1rPYv38/69evr3ZoZoWMmywkNZFVP82VNJus+gngdGBBBWIzm1L27dvHnj17AHjssccOq5Iyq3UTtVmsBj4JvA74fyVfPQvcEhF/Xf7wjo3bLKzWzJw5k/379zN79mz27t176LO5uZlf/9qLT1ptONY2i/URsRj404hYXLL9Zi0nCrNatH//fmbOnMmdd97JSy+9xJ133nkogZjVgyJdZwclfeCI7TxJr53oIkkLJfVIeljSL9KbCpLmSNoqaUf6nJ3KJelLknZK+pmkc0rutSKdv0PSiuP6xWZVsmbNGjo6OmhsbKSjo4M1a9ZUOySzwsathjp0gvRd4B3AWIfwdwPbgMXADRHxtXGuOwM4IyIeknRauuYS4KPAUER8XtIaYHZEXJfGcqwCLgKWAusjYqmkOUAf0E7WlXcb8LY0dfpRuRrKas1EkwZ6tTyrFcdUDVWiEWiNiA9GxAeBs8n+0V4KjDuyOyKeiIiH0v5zQD9Zw/jFwOZ02mayBEIqvzUyDwCzUsK5ANgaEUMpQWwFLiwQt1nNGEsWY2tuj3165lmrF0WSRUtEPFVyvBtYGBFDZIP0ckk6C/gt4MfA/Ih4In31JDA/7S8AdpVcNpDKxis3qxsRgaRDA/BGR0eR5LcKqxtF5oa6X9LdwDfT8QdTWTOwL+9iSTPJlmP9ZEQ8W/o/qYgISSfkb4uklcBKgEWLFp2IW5qdUA0NDRw4cGDcY7NaVuTN4hrgq8Bb03YrcE1E7I+ICedYltRIlihui4hvp+KnUvXSWLvG7lQ+CCwsubwllY1XfpiI2BAR7RHRPm/evAI/y6yyDhw4wNVXX82+ffu4+uqrnSisruQ2cB/zjbNXiM1kjdmfLCn/r8DTJQ3ccyLiU5J+D/hjXm7g/lJEnJsauLcBY72jHiJr4B531Rg3cFutGXujPu2009i/fz/Nzc0899xzgBu4rXZM1MBdZLqPDwB/DryWbBR30YkE3wX8EfBzST9NZZ8GPg/cIakTeBy4NH13D1mi2Ak8D1xF9qAhSTcCD6bzbpgoUZjVqmnTph1KEM899xzTpk3z24XVjSJdZ3cCfxAR/ZUJ6fj5zcJqjSQaGho45ZRTGBkZobGxkYMHDzI6Ouo3C6sZx/VmATxVT4nCrFaNjo4e6g01MlKoI6FZzSiSLPok3Q78LdmaFgCUNFibmdkUVyRZnE7WhnB+SVkAThZmk9TU1MTw8PChT7N6kZssIuKqSgRidjIYSxBOFFZvcsdZSHqjpPskbU/Hb5H0mfKHZmZmtaLIoLxbgOtJU3tExM+Ay8sZlJmZ1ZYiyWJGRPzDEWXuHG5mdhIpkix+JekNZI3aSPoQ8MTEl5iZ2VRSpDfUNcAG4M2SBoFHgSvKGpWZmdWU3DeLiHgkIn4XmAe8OSKWAe8ve2RmZlYzilRDAZBmmX0uHV5bpnjMzKwGFU4WR/DyXmZmJ5FjTRae+czM7CQybgO3pOc4elIQ8KqyRWRmZjVn3GQREadVMhAzM6tdx1oNZWZmJxEnCzMzy+VkYWZmuZwszMwsl5OFmZnlKluykLRJ0u6xdTBS2RxJWyXtSJ+zU7kkfUnSTkk/k3ROyTUr0vk7JK0oV7xmZja+cr5ZfBW48IiyNcB9EbEEuC8dA7wXWJK2lcDNkCUXYC2wFDgXWDuWYMzMrHLKliwi4kfA0BHFFwOb0/5m4JKS8lsj8wAwS9IZwAXA1ogYioi9wFZemYDMzKzMKt1mMT8ixtbCeBKYn/YXALtKzhtIZeOVm5lZBVWtgTsighM4x5SklZL6JPXt2bPnRN3WzMyofLJ4KlUvkT53p/JBYGHJeS2pbLzyV4iIDRHRHhHt8+bNO+GBm5mdzCqdLLYAYz2aVgB3lZRfmXpFvR14JlVX3QucL2l2atg+P5WZmVkFFVlW9ZhI6gbeDcyVNEDWq+nzwB2SOoHHgUvT6fcAFwE7geeBqwAiYkjSjcCD6bwbIuLIRnMzMyszZU0HU0t7e3v09fVVOwyzQ6Tx1wubin8HrT5J2hYR7Uf7ziO4zcwsl5OFmZnlcrIwM7NcThZmZpbLycLMzHI5WZiZWS4nCzMzy+VkYWZmuZwszMwsl5OFmZnlcrIwM7NcThZmZpbLycLMzHI5WZiZWS4nCzMzy+VkYWZmuZwszMwsV9mWVTU7GUy0At6JvIdX07Nqc7IwOw5F/xH3sqpW71wNZWZmueomWUi6UNI/SdopaU214zGbjPHeHvxWYfWiLpKFpAbgvwPvBc4Glks6u7pR2VQ0Z84cJJVlO5pyPWvOnDkV/pOzqa5e2izOBXZGxCMAkr4BXAw8XNWobMrZu3fvlPjf/oloeDcrVS/JYgGwq+R4AFhapVhsCou1p8NnX13tMI5brD292iHYFFMvySKXpJXASoBFixZVORqrW599ZlKnV+p/8FPhbcfqW120WQCDwMKS45ZUdkhEbIiI9ohonzdvXkWDs5NXRFRkM6u2ekkWDwJLJC2WNB24HNhS5ZjMzE4adVENFREHJP0xcC/QAGyKiF9UOSwzs5NGXSQLgIi4B7in2nGYmZ2M6qUayszMqsjJwszMcjlZmJlZLicLMzPL5WRhZma5NBUH/EjaAzxe7TjMxjEX+FW1gzA7ijMj4qijmqdksjCrZZL6IqK92nGYTYaroczMLJeThZmZ5XKyMKu8DdUOwGyy3GZhZma5/GZhZma5nCzMKkTSJkm7JW2vdixmk+VkYVY5XwUurHYQZsfCycKsQiLiR8BQteMwOxZOFmZmlsvJwszMcjlZmJlZLicLMzPL5WRhViGSuoG/B94kaUBSZ7VjMivKI7jNzCyX3yzMzCyXk4WZmeVysjAzs1xOFmZmlsvJwszMcjlZmJlZLicLMzPL5WRhZma5/j/a8krIYvaelQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Clean RAM\n",
        "lines_eng = lines_rus = None"
      ],
      "metadata": {
        "id": "7UBCOLHqAxG9"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Filter too long and too small *phrases* and bad ratio"
      ],
      "metadata": {
        "id": "NBmwl80uwlNp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "l = stats_hard_filtered[1]['len']\n",
        "min_q = np.quantile(l['eng'], 0.35)\n",
        "max_q = np.quantile(l['eng'], 0.65)\n",
        "print(f'min_q {min_q}, max_q {max_q}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ukhVnQNk-M8r",
        "outputId": "0b80b466-357d-4d7f-a1ed-b8094bef307d"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "min_q 29.0, max_q 76.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "min_ratio = 1 / 2 \n",
        "max_ratio = 2\n",
        "ratious = stats_hard_filtered[1]['ratious']\n",
        "\n",
        "bad_ratio_idxs = np.where((ratious < min_ratio) |\n",
        "                           (ratious > max_ratio))[0]\n",
        "print(f'Pairs with bad ratio: {len(bad_ratio_idxs)}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X23QR04Dk56F",
        "outputId": "3ffc22b4-78ec-4ad5-dcb2-a489f7e1b119"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pairs with bad ratio: 715850\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_pair(idx):\n",
        "    return filtered_lines['eng'][idx], filtered_lines['rus'][idx] \n",
        "\n",
        "\n",
        "np.random.seed(42)\n",
        "print('Pairs with bad ratio examples:')\n",
        "for idx in np.random.choice(bad_ratio_idxs, 10):\n",
        "    print(get_pair(idx))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oh-0rLColff2",
        "outputId": "98562729-411c-4d4d-9aa8-fd4e32316fe3"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pairs with bad ratio examples:\n",
            "('Страницы: [1]\\n', 'Страницы: 1 ... 23 24 [25] 26 27 ... 249\\n')\n",
            "('10\\n', 'Фотографии Т-50 (10)\\n')\n",
            "('Branches and representative offices of international and foreign non-governmental organisations shall be governed in their activity by the Charters of such international and foreign non-governmental organisations, provided that such Charters are consistent with the legislation of the Republic of Kazakhstan.\\n', 'Лица, виновные в нарушении настоящего Закона, несут ответствен-ность в соответствии с законами Республики Казахстан.\\n')\n",
            "('All the cartoonists whose works will be selected for the catalogue will receive a catalogue.\\n', '3. По итогам конкурса будет издан каталог.\\n')\n",
            "('Related tags: high-tech, strategies, successful\\n', 'Related tags: деньги, успех, финансы, управление, менеджмент, саморазвитие, кийосаки, предприятие, богатство, маркетинг\\n')\n",
            "('After brilliant performance in Budapest dzhuybarsky wrestlers met in the hometown as national heroes. 11 years later the national team of Iran won again the World Cup, and an important role in this victory played пехлеваны from Dzhuybar.\\n', 'Спустя 11 лет сборная Ирана вновь выиграла чемпионат мира, и важную роль в этой победе сыграли пехлеваны из Джуйбара.\\n')\n",
            "('During the application care has to be taken to spread the adhesive in fine layer in order to glue the pearls to the wax structure without dipping them into the adhesive.\\n', 'Перлы должны прикрепляться к поверхности и не погружаться в клей.\\n')\n",
            "('Avatud:\\n', 'Источник: stragendo.ee\\n')\n",
            "(\"This data has no connection to a user's personal data in any way.\\n\", 'Мы производим сбор не являющихся личными данных о пользовании нашим сайтом в статистических целях. . Эти данные не имеют никакого отношения к личным данным пользователя.\\n')\n",
            "('12-12-2017, 13:09\\n', 'Пароль:\\n')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "good_ri = set(range(len(ratious))) - set(bad_ratio_idxs)\n",
        "\n",
        "ratious[list(good_ri)].max()\n",
        "for key in ['eng', 'rus']:\n",
        "    filtered_lines[key] = [sent for idx, sent in enumerate(filtered_lines[key]) if idx in good_ri]\n",
        "\n",
        "print(f'{len(bad_ratio_idxs)} elements was removed')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TDX_-chr17lm",
        "outputId": "6ec896e3-6688-423c-f85b-5f8a526a9a20"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "715850 elements was removed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outlayers_idxs = {idx for lang in ['rus', 'eng']\n",
        "                  for idx, data in enumerate(filtered_lines[lang])\n",
        "                  if len(data) < min_q  or len(data) > max_q}\n",
        "\n",
        "#outlayers_idxs.union(set(bad_ratio_idxs))\n",
        "\n",
        "filtered_lines_filter_q = {}\n",
        "for key, data in filtered_lines.items(): \n",
        "    filtered_lines_filter_q[key] = [sent for idx, sent in enumerate(data) \n",
        "                                    if idx not in outlayers_idxs]\n",
        "\n",
        "print(f'{len(outlayers_idxs)} elements was removed')"
      ],
      "metadata": {
        "id": "fyit018cyJnP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4080efdf-b1bd-42b1-ab12-5d91a2c80625"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "7975256 elements was removed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stat_filtered_q = get_statistics(filtered_lines_filter_q['rus'], \n",
        "                                 filtered_lines_filter_q['eng'])\n",
        "print(stat_filtered_q[0])\n",
        "plot_counts_boxplot(stat_filtered_q)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "z1YVZFGG0_lj",
        "outputId": "eecf388b-df97-4760-ecfd-d6e9391ebcbf"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Lenght statistics:\n",
            "Rus:{'mean': 46.19948509146689, 'argmin': 83, 'argmax': 450, 'min': 29, 'max': 76}\n",
            "Eng{'mean': 46.29208094422188, 'argmin': 49, 'argmax': 176, 'min': 29, 'max': 76}\n",
            "Rus / Eng {'mean': 1.0159408232277127, 'argmin': 3442, 'argmax': 2570, 'min': 0.5, 'max': 2.0}\n",
            "Pairs\n",
            "Smallest: ('4. Parishes and Communities.\\n', '4. Приходы и приходские общины.\\n')\n",
            "Longest: ('You will find the booking class on your ticket: where is the booking class?\\n', 'Класс бронирования указан в Вашем билете: где указан класс бронирования?\\n')\n",
            "Smallest ratio: ('Below is the tungsten alloy radiation shielding terms and definition.\\n', 'Радиационные термины и определения\\n')\n",
            "Biggest ratio: (\"See also: I'd , I'm , i's , ism\\n\", 'See also: chat , ghat , hat , that , what , wheat , whet , whit\\n')\n",
            "Counts of pairs: 2931006\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAP9klEQVR4nO3df6xfdX3H8edrVMKPCbT22jEqtiLClkXQ3aiAMQpinD8Q0RkXt3SOiGaGYZhTNBrQ+QeaTEdc4iw6V4hTBH9UYWFiZ3Umm+wWURE01U6UBtoLFIq6tKLv/fE9d1zbe2/Pve353n57no/km+/3c77fc867/7zu6ed8zueTqkKS1B+/tdgFSJKGy+CXpJ4x+CWpZwx+SeoZg1+SembJYhfQxvLly2vVqlWLXYYkjZRNmzbdX1Vje24fieBftWoVExMTi12GJI2UJHfPtN2uHknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ4x+CWpZ0biAS5pWJIM5Tyug6HFZPBL08w3kJMY4ho5dvVIUs8Y/JLUMwa/JPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k9Y/BLUs8Y/JLUMwa/JPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST3TWfAnOSXJ7dNeO5O8JcmyJLck2dy8L+2qBknS3joL/qr6QVWdXlWnA38I/AL4PHAZsKGqTgY2NG1J0pAMq6vnHOBHVXU38ApgXbN9HXD+kGqQJDG84H8t8Knm84qqurf5fB+wYqYdklyUZCLJxOTk5DBqlKRe6Dz4kxwOnAdcv+d3VVVAzbRfVa2tqvGqGh8bG+u4Sknqj2Fc8f8RcFtVbWva25IcD9C8bx9CDZKkxjCC/094rJsH4IvAmubzGmD9EGqQJDU6Df4kRwPnAp+btvlK4Nwkm4EXNm1J0pAs6fLgVfVz4Al7bHuAwSgfSdIi8MldSeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ7ZZ/AnOSrJu5Nc3bRPTvKy7kuTJHWhzRX/J4BdwBlNeyvwvs4qkiR1qk3wn1RVHwB+CVBVvwDSaVWSpM60Cf7dSY6kWSIxyUkM/gcgSRpBbebjvxy4GXhSkk8CZwF/3mVRkqTu7DP4q+qWJLcBz2HQxXNJVd3feWWSpE60GdXzSuDRqrqpqm4EHk1yfvelSZK60KaP//KqeniqUVUPMej+kSSNoDbBP9NvOl2rV5LUnTbBP5Hkg0lOal4fBDZ1XZgkqRttgv9iYDdwXfPaBby5y6IkSd1pM6rn58BlQ6hFkjQE+wz+JE8D3gqsmv77qjq7u7IkSV1pc5P2euAfgY8Bv+q2HElS19oE/6NV9ZHOK5EkDUWbm7tfSvKXSY5Psmzq1XllkqROtLniX9O8/820bQU85cCXI0nqWptRPauHUYgkaTjarsD1riRrm7YrcEnSCGu7Atdu4Mym7QpckjTCXIFLknrGFbgkqWfaBP8V/OYKXBuAt7c5eJLjktyQ5PtJ7kpyRjMc9JYkm5v3pQsvX5I0X/sM/qr6MnABg+UWPwWMV9VXWx7/KuDmqjoVOA24i8G8Pxuq6mQGf0ScB0iShqjNqJ4NVfXA1ApcVXV/kg0t9jsWeB7wcYCq2t0s4vIKYF3zs3WAq3lJ0hDNOo4/yRHAUcDypjtm6obuMcAJLY69GpgEPpHkNAZz+F8CrKiqe5vf3AesWGDtkqQFmOuK/40MwvrU5n3qtR74hxbHXgI8E/hIVT0D2Gt656oqmpvGe0pyUZKJJBOTk5MtTidJamPW4K+qq5qndt9aVU+pqtXN67SqahP89wD3VNU3m/YNDP4QbEtyPEDzvn2W86+tqvGqGh8bG5vXP0qSNLs2UzZ8OMmZ7D0f/zX72O++JD9NckpV/QA4B7izea0Brmze1y+8fEnSfLVZiOVa4CTgdh6bj7+AOYO/cTHwySSHA1uA1zP4X8ZnklwI3A28ZgF1S5IWqM3snOPA7zf98fNSVbc3++/pnPkeS5J0YLR5gOsO4He6LkSSNBxtrviXA3cmuZVpUzVU1XmdVSVJ6kyb4L+i6yIkScPTZlTP15I8GTi5qr6S5CjgsO5LkyR1oc2UDW9gMAb/o82mE4AvdFmUJKk7bW7uvhk4C9gJUFWbgSd2WZQkqTttgn9XVe2eaiRZwizTLEiSDn5tgv9rSd4JHJnkXOB64EvdliVJ6kqb4L+MwSyb32Uwcdu/Au/qsihJUnfajOr5NXA1cHWSZcDKhTzFK0k6OLQZ1bMxyTFN6G9i8AfgQ92XJknqQpuunmOraieD5Revqapn41w7kjSy2gT/kmbe/NcAN3ZcjySpY22C/73AvwE/rKr/TvIUYHO3ZUmSutLm5u71DIZwTrW3AK/qsihJUnfaXPFLkg4hBr8k9YzBL0k902bN3Utn2PwwsKlZWlGSNELaXPGPA29iMB3zCQymbXgxgwe53tZhbZKkDrRZgWsl8Myq+hlAksuBm4DnMXiS9wPdlSct3LJly9ixY0fn50nS6fGXLl3Kgw8+2Ok51C9tgv+JTFtrF/glsKKq/jfJrln2kRbdjh07OBSmler6D4v6p03wfxL4ZpL1TfvlwL8kORq4s7PKJEmdaPMA198muRk4s9n0pqqaaD6/rrPKJEmdaHPFD3AbsHXq90lOrKqfdFaVJKkzbYZzXgxcDmwDfgWEwdKLT++2NElSF9pc8V8CnFJVD3RdjCSpe23G8f+UwQNbkqRDQJsr/i3AxiQ3MW1YZ1V9sLOqJEmdaRP8P2lehzcvSdIIazOc8z17bkvSdjSQJOkgM2sff5JvTPt87R5f39pZRZKkTs11c/foaZ//YI/vfIZckkbUXF02NcvnmdozSvJj4BEG4/8frarxJMuA64BVwI+B11RV9zNpSZKAuYP/uCSvZPC/guOSXNBsD3DsPM7xgqq6f1r7MmBDVV2Z5LKm/fb5FC1JWri5gv9rwHnTPr982ndf349zvgJ4fvN5HbARg1+ShmbW4K+q1x+A4xfw5SQFfLSq1jKY0vne5vv7gBUH4DySpJa6Hpb53KramuSJwC1Jvj/9y6qq5o/CXpJcBFwEcOKJJ3ZcpiT1R6eLrVfV1uZ9O/B54FnAtiTHAzTv22fZd21VjVfV+NjYWJdlSlKvzDWO/4+b99ULOXCSo5M8fuoz8CLgDuCLwJrmZ2uA9TMfQZLUhbmu+N/RvH92gcdeAXwjybcZPPB1U1XdDFwJnJtkM/DCpi1JGpK5+vgfSPJlYHWSL+75ZVWdN8M+07/fApw2w/YHgHPmW6gk6cCYK/hfCjwTuBb4u+GUI0nq2lzDOXcD/5XkzKqaTPLbzfafDa06SdIB12ZUz4ok3wK+B9yZZFOSPefukSSNiDbBvxa4tKqeXFUnAn/dbJMkjaA2wX90VX11qlFVG/nNmTslSSOk1dKLSd7N4CYvwJ8yWI5RkjSC2lzx/wUwBnyOwZj+5c02SdIIarP04g7gr4ZQiyRpCDqdq0eSdPAx+CWpZwx+SeqZffbxJxkD3sBgjdz//31VeYNXkkZQm+Gc64H/AL7CYNF0SdIIaxP8R1WVa+JK0iGiTR//jUle0nklkqShmPWKP8kjDBZLD/DOJLuAXzbtqqpjhlOiJOlAmmta5scPsxBJ0nDss6snyYY22yRJo2Gurp4jGMzCuTzJUgZdPADHACcMoTZJUgfmGtXzRuAtwO8Ct03bvhP4hy6LkiR1Z64+/quAq5JcXFUfHmJN0gFRlx8DVxy72GXst7rccRQ6sNqM49+a5II9tj0MfLeqtndQk3RA5D07qarFLmO/JaGuWOwqdChpE/wXAmcAU6twPR/YBKxO8t6quna2HSVJB582wf844PeqahtAkhXANcCzga/z2MpckqQR0ObJ3ZVTod/YDjypqh5k8ECXJGmEtLni35jkRuD6pv2qZtvRwEOdVSZJ6kSb4H8zg7A/q2lfA3y2BnfNXtBVYZKkbrRZc7eAG5qXJGnEtZmy4YIkm5M8nGRnkkeS7BxGcZKkA69NV88HgJdX1V1dFyNJ6l6bUT3bDH1JOnS0ueKfSHId8AVg19TGqvpcZ1VJkjrTJviPAX4BvGjatgIMfkkaQW1G9bx+f06Q5DBgAthaVS9Lshr4NPAEBlM//FlV7d6fc0iS2mszqudpSTYkuaNpPz3Ju+ZxjkuA6fcI3g98qKqeCuxgMBeQJGlI2tzcvRp4B830DFX1HeC1bQ6eZCXwUuBjTTvA2Tz2TMA64Pz5lSxJ2h9tgv+oqrp1j22Ptjz+3wNvA37dtJ8APFRVU/vfg6t5SdJQtQn++5OcxOCGLkleDdy7r52SvAzYXlWbFlJYkouSTCSZmJycXMghJEkzaDtXz1rg1CRbgf8BXtdiv7OA85K8BDiCweigq4DjkixprvpXAltn2rmq1jbnZXx8fPRX05Ckg8Q+r/iraktVvRAYA06tqucCr2yx3zuqamVVrWJwT+Dfq+p1DBZ0eXXzszXA+oUWL0mavzZdPQBU1c+r6pGmeel+nPPtwKVJfsigz//j+3EsSdI8tenqmUnm8+Oq2ghsbD5vAZ61wPNKkvZT6yv+PdjnLkkjatYr/iSPMHPABziys4okSZ2aNfir6vHDLESSNBwL7eqRJI0og1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnFjpXjzQSBou+jbalS5cudgk6xBj8OmRVdT+lVJKhnEc6kOzqkaSeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ7pLPiTHJHk1iTfTvK9JO9ptq9O8s0kP0xyXZLDu6pBkrS3Lq/4dwFnV9VpwOnAi5M8B3g/8KGqeiqwA7iwwxokSXvoLPhr4GdN83HNq4CzgRua7euA87uqQZK0t077+JMcluR2YDtwC/Aj4KGqerT5yT3ACbPse1GSiSQTk5OTXZYpSb3SafBX1a+q6nRgJfAs4NR57Lu2qsaranxsbKyzGiWpb4YyqqeqHgK+CpwBHJdkSfPVSmDrMGqQJA10OapnLMlxzecjgXOBuxj8AXh187M1wPquapAk7W3Jvn+yYMcD65IcxuAPzGeq6sYkdwKfTvI+4FvAxzusQZqXJEPZp6rmvY90oHQW/FX1HeAZM2zfwqC/XzroGMjqA5/claSeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6JqPwwEqSSeDuxa5DmsFy4P7FLkKaxZOraq9ZLkci+KWDVZKJqhpf7Dqk+bCrR5J6xuCXpJ4x+KX9s3axC5Dmyz5+SeoZr/glqWcMfknqGYNfWoAk/5Rke5I7FrsWab4Mfmlh/hl48WIXIS2EwS8tQFV9HXhwseuQFsLgl6SeMfglqWcMfknqGYNfknrG4JcWIMmngP8ETklyT5ILF7smqS2nbJCknvGKX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWf+DzgBEUjpudHvAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "GT: ('Top of performance 85 %\\n', 'Максимальная яйценоскость 85 %\\n'), interesting!\n",
        "\n",
        "Looks like non translated outlayer appears in dataset, let's filter em"
      ],
      "metadata": {
        "id": "m0-d8aE_3YTt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "russian_alph = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'\n",
        "eng_alph = 'abcdefghigklmnopqrstuvwxyz'\n",
        "\n",
        "any_russian_ch = lambda x: any(rus_ch in x for rus_ch in russian_alph)\n",
        "any_eng_ch = lambda x: any(eng_ch in x for eng_ch in eng_alph) \n",
        "\n",
        "non_translated_idxs = {idx for idx, data in enumerate(filtered_lines_filter_q['rus'])\n",
        "                       if not any_russian_ch(data)}\n",
        "language_mixed = {idx for idx, data in enumerate(filtered_lines_filter_q['eng'])\n",
        "                       if any_russian_ch(data)}.union({\n",
        "                           idx for idx, data in enumerate(filtered_lines_filter_q['rus'])\n",
        "                           if any_eng_ch(data)\n",
        "                       })\n",
        "                    \n",
        "outlayers = non_translated_idxs.union(language_mixed)\n",
        "print(f'{len(outlayers)} non translated pairs was found')"
      ],
      "metadata": {
        "id": "zibrdCA841xO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e94dfaf9-734f-4ae6-ffc1-486bd2894d23"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1259522 non translated pairs was found\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_pair(idx):\n",
        "    return filtered_lines_filter_q['eng'][idx], filtered_lines_filter_q['rus'][idx] \n",
        "\n",
        "\n",
        "np.random.seed(42)\n",
        "print('Not translated pairs example')\n",
        "for idx in np.random.choice(list(non_translated_idxs), 10):\n",
        "    print(get_pair(idx))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eOiyjoySripI",
        "outputId": "a694064d-c666-49ac-cc7f-beb9c5954f6d"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Not translated pairs example\n",
            "('« Reply #1 on: March 02, 2009, 04:25:53 PM »\\n', '« Reply #1 on: December 02, 2005, 11:04:34 AM »\\n')\n",
            "('#24 Encuentro desconocido (00:00:43)\\n', '#28 Encuentro desconocido (00:01:45)\\n')\n",
            "('« Reply #2 on: August 30, 2008, 10:52:13 AM »\\n', '« Reply #2 on: September 20, 2007, 08:59:31 AM »\\n')\n",
            "('This page has been generated on Saturday, Jun 4th 2016, 14:30 pm CST\\n', 'This page has been generated on Thursday, Apr 14th 2016, 22:58 pm CST\\n')\n",
            "('« Reply #19 on: August 03, 2009, 12:42:24 PM »\\n', '« Reply #3 on: January 30, 2009, 10:09:08 AM »\\n')\n",
            "('17.04.2012 15:40 (0 comments)\\n', '17.04.2012 15:40 (0 comments)\\n')\n",
            "('Poem Submitted: Tuesday, March 30, 2010\\n', 'Poem Submitted: Friday, September 11, 2015\\n')\n",
            "('This list was generated on Sun Jun 12 10:24:31 2016 QYZT.\\n', 'This list was generated on Sun Jun 12 10:01:21 2016 QYZT.\\n')\n",
            "('Play other great free collecting.\\n', 'Play other great free collecting .\\n')\n",
            "('Gare de l’Est (metro line 5)\\n', 'Gare de l’Est (direct métro ligne 5)\\n')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "filtered_final = {}\n",
        "for key, data in filtered_lines_filter_q.items(): \n",
        "    filtered_final[key] = [sent for idx, sent in enumerate(data) \n",
        "                                    if idx not in outlayers]\n",
        "\n",
        "print(f'{len(outlayers)} elements was removed')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "221htrQlsBYs",
        "outputId": "fc653f24-f642-4f63-f700-530fa720c0cf"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1259522 elements was removed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stat_filtered_final = get_statistics(filtered_final['rus'], \n",
        "                                     filtered_final['eng'])\n",
        "print(stat_filtered_final[0])\n",
        "plot_counts_boxplot(stat_filtered_final)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "kLvb1nIS66Z3",
        "outputId": "74cd322f-d9e7-4f98-8e3d-fd7c2b824d94"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Lenght statistics:\n",
            "Rus:{'mean': 46.955562242893144, 'argmin': 68, 'argmax': 274, 'min': 29, 'max': 76}\n",
            "Eng{'mean': 47.32550057314339, 'argmin': 35, 'argmax': 148, 'min': 29, 'max': 76}\n",
            "Rus / Eng {'mean': 1.0158406613946556, 'argmin': 1079, 'argmax': 1267, 'min': 0.5, 'max': 2.0}\n",
            "Pairs\n",
            "Smallest: ('4. Parishes and Communities.\\n', '4. Приходы и приходские общины.\\n')\n",
            "Longest: ('You will find the booking class on your ticket: where is the booking class?\\n', 'Класс бронирования указан в Вашем билете: где указан класс бронирования?\\n')\n",
            "Smallest ratio: ('Below is the tungsten alloy radiation shielding terms and definition.\\n', 'Радиационные термины и определения\\n')\n",
            "Biggest ratio: ('Cancer (3) Apply filter: Cancer\\n', 'Психическое здоровье (3) Применить фильтр: Психическое здоровье\\n')\n",
            "Counts of pairs: 1671484\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAP+klEQVR4nO3df6xfdX3H8edrVMKPCbT22jEqtiLClkXQ3aiAMQpinD8Q0RkXt3SOiEbDMMwpGg3o/ANNpiOaOAvOFeIUwR9VWJjYWZ3JJrtFVARNtROlgfYChaIuVPS9P76n43p77+25tz3f22/P85F88/2e8/2ec97953U//ZzP+XxSVUiS+uN3FrsASdJwGfyS1DMGvyT1jMEvST1j8EtSzyxZ7ALaWL58ea1atWqxy5CkkbJp06b7qmps+v6RCP5Vq1YxMTGx2GVI0khJctdM++3qkaSeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6ZiQe4JKGJclQruM6GFpMBr80xXwDOYkhrpFjV48k9YzBL0k9Y/BLUs8Y/JLUMwa/JPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k9Y/BLUs8Y/JLUMwa/JPWMwS9JPWPwS1LPGPyS1DOdBX+Sk5LcNuW1M8lbkyxLcnOSzc370q5qkCTtqbPgr6ofVtWpVXUq8MfAL4EvAJcAG6rqRGBDsy1JGpJhdfWcBfy4qu4CXgGsa/avA84dUg2SJIYX/K8FPt18XlFV9zSf7wVWzHRAkguSTCSZmJycHEaNktQLnQd/kkOBc4Drpn9XVQXUTMdV1dqqGq+q8bGxsY6rlKT+GEaL/0+AW6tqW7O9LcmxAM379iHUIElqDCP4/4zHunkAvgSsaT6vAdYPoQZJUqPT4E9yJHA28Pkpuy8Hzk6yGXhhsy1JGpIlXZ68qn4BPGHavvsZjPKRJC0Cn9yVpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6pm9Bn+SI5K8J8mVzfaJSV7WfWmSpC60afF/EngEOK3Z3gq8v7OKJEmdahP8J1TVB4FfAVTVL4F0WpUkqTNtgn9XksNplkhMcgKD/wFIkkZQm/n4LwVuAp6U5FPAGcBfdlmUJKk7ew3+qro5ya3Acxh08VxUVfd1XpkkqRNtRvW8Eni0qm6sqhuAR5Oc231pkqQutOnjv7SqHtq9UVUPMuj+kSSNoDbBP9NvOl2rV5LUnTbBP5HkQ0lOaF4fAjZ1XZgkqRttgv9CYBdwbfN6BHhLl0VJkrrTZlTPL4BLhlCLJGkI9hr8SZ4GvA1YNfX3VXVmd2VJkrrS5ibtdcA/AlcBv+62HElS19oE/6NV9bHOK5EkDUWbm7tfTvLmJMcmWbb71XllkqROtGnxr2ne/3bKvgKesv/LkSR1rc2ontXDKESSNBxtV+B6d5K1zbYrcEnSCGu7Atcu4PRm2xW4JGmEuQKXJPWMK3BJUs+0Cf7L+O0VuDYA72hz8iTHJLk+yQ+S3JnktGY46M1JNjfvSxdeviRpvvYa/FX1FeA8BsstfhoYr6qvtTz/FcBNVXUycApwJ4N5fzZU1YkM/og4D5AkDVGbUT0bqur+3StwVdV9STa0OO5o4HnAJwCqaleziMsrgHXNz9YBruYlSUM06zj+JIcBRwDLm+6Y3Td0jwKOa3Hu1cAk8MkkpzCYw/8iYEVV3dP85l5gxQJrlyQtwFwt/jcyCOuTm/fdr/XAR1ucewnwTOBjVfUMYI/pnauqaG4aT5fkgiQTSSYmJydbXE6S1MaswV9VVzRP7b6tqp5SVaub1ylV1Sb47wburqpvNdvXM/hDsC3JsQDN+/ZZrr+2qsaranxsbGxe/yhJ0uzaTNnwkSSns+d8/Ffv5bh7k/wsyUlV9UPgLOCO5rUGuLx5X7/w8iVJ89VmIZZrgBOA23hsPv4C5gz+xoXAp5IcCmwBXs/gfxmfTXI+cBfwmgXULUlaoDazc44Df9j0x89LVd3WHD/dWfM9lyRp/2jzANftwO91XYgkaTjatPiXA3ckuYUpUzVU1TmdVSVJ6kyb4L+s6yIkScPTZlTP15M8GTixqr6a5AjgkO5LkyR1oc2UDW9gMAb/482u44AvdlmUJKk7bW7uvgU4A9gJUFWbgSd2WZQkqTttgv+Rqtq1eyPJEmaZZkGSdOBrc3P360neBRye5GzgzcCXuy1L2nfLli1jx44dnV8n6XZBuqVLl/LAAw90eg31S5vgvwQ4H/geg4nb/hW4qsuipP1hx44dLOC5wwNO139Y1D9tRvX8BrgSuDLJMmDlQp7ilSQdGNqM6tmY5Kgm9Dcx+APw4e5LkyR1oc3N3aOraieD5Revrqpn41w7kjSy2gT/kmbe/NcAN3RcjySpY22C/33AvwE/qqr/TvIUYHO3ZUmSutLm5u51wHVTtrcAr+qyKElSd9q0+CVJBxGDX5J6xuCXpJ5ps+buxTPsfgjY1CytKEkaIW1a/OPAmxhMx3wcg2kbXszgQa63d1ibJKkDbebqWQk8s6p+DpDkUuBG4HkMnuT9YHflSZL2tzYt/icyZa1d4FfAiqr632n7JUkjoE2L/1PAt5Ksb7ZfDvxLkiOBOzqrTJLUiTYPcP1dkpuA05tdb6qqiebz6zqrTJLUiTYtfoBbga27f5/k+Kr6aWdVSZI602Y454XApcA24NdAGCy9+PRuS5MkdaFNi/8i4KSqur/rYiRJ3WszqudnDB7YkiQdBNq0+LcAG5PcyJThm1X1oc6qkiR1pk3w/7R5Hdq8JEkjrM1wzvdO35ek7WggSdIBZtY+/iTfnPL5mmlf39JZRZKkTs11c/fIKZ//aNp36aAWSdIQzNVlU7N8nml7Rkl+AjzMYPz/o1U1nmQZcC2wCvgJ8Jqq2tGyXknSPpor+I9J8koG/ys4Jsl5zf4AR8/jGi+oqvumbF8CbKiqy5Nc0my/Yz5FS5IWbq7g/zpwzpTPL5/y3Tf24ZqvAJ7ffF4HbMTgl6ShmTX4q+r1++H8BXwlSQEfr6q1DKZ0vqf5/l5gxX64jiSppa6HZT63qrYmeSJwc5IfTP2yqqr5o7CHJBcAFwAcf/zxHZcpSf3R6WLrVbW1ed8OfAF4FrAtybEAzfv2WY5dW1XjVTU+NjbWZZmS1CtzjeP/0+Z99UJOnOTIJI/f/Rl4EXA78CVgTfOzNcD6mc8gSerCXC3+dzbvn1vguVcA30zyHQYPfN1YVTcBlwNnJ9kMvLDZliQNyVx9/Pcn+QqwOsmXpn9ZVefMcMzU77cAp8yw/37grPkWKknaP+YK/pcCzwSuAf5+OOVIkro213DOXcB/JTm9qiaT/G6z/+dDq06StN+1GdWzIsm3ge8DdyTZlGT63D2SpBHRJvjXAhdX1ZOr6njgb5p9kqQR1Cb4j6yqr+3eqKqN/PbMnZKkEdJq6cUk72FwkxfgzxksxyhJGkFtWvx/BYwBn2cwpn95s0+SNILaLL24A/jrIdQiSRqCTufqkSQdeAx+SeoZg1+SemavffxJxoA3MFgj9/9/X1Xe4JWkEdRmOOd64D+ArzJYNF2SNMLaBP8RVeWauJJ0kGjTx39Dkpd0XokkaShmbfEneZjBYukB3pXkEeBXzXZV1VHDKVGStD/NNS3z44dZiLS/1aVHwWVHL3YZ+6wutY2l/avNqJ4NVXXW3vZJB5q8dydVtdhl7LMk1GWLXYUOJnN19RzGYBbO5UmWMujiATgKOG4ItUmSOjBXi/+NwFuB3wdunbJ/J/DRLouSJHVnrj7+K4ArklxYVR8ZYk2SpA61Gce/Ncl50/Y9BHyvqrZ3UJMkqUNtgv984DRg9ypczwc2AauTvK+qrpntQEnSgadN8D8O+IOq2gaQZAVwNfBs4Bs8tjKXJGkEtHlyd+Xu0G9sB55UVQ8weKBLkjRC2rT4Nya5Abiu2X5Vs+9I4MHOKpMkdaJN8L+FQdif0WxfDXyuBk/GvKCrwiRJ3Wiz5m4B1zcvSdKI22sff5LzkmxO8lCSnUkeTrJzGMVJkva/Nl09HwReXlV3dl2MJKl7bUb1bDP0Jeng0abFP5HkWuCLwCO7d1bV5zurSpLUmTbBfxTwS+BFU/YVYPBL0ghqM6rn9ftygSSHABPA1qp6WZLVwGeAJzCY+uEvqmrXvlxDktRem1E9T0uyIcntzfbTk7x7Hte4CJh6j+ADwIer6qnADgZzAUmShqTNzd0rgXfSTM9QVd8FXtvm5ElWAi8Frmq2A5zJY88ErAPOnV/JkqR90Sb4j6iqW6bte7Tl+f8BeDvwm2b7CcCDVbX7+LtxNS9JGqo2wX9fkhMY3NAlyauBe/Z2UJKXAduratNCCktyQZKJJBOTk5MLOYUkaQZt5+pZC5ycZCvwP8DrWhx3BnBOkpcAhzEYHXQFcEySJU2rfyWwdaaDq2ptc13Gx8dHf8VsSTpA7LXFX1VbquqFwBhwclU9F3hli+PeWVUrq2oVg3sC/15Vr2OwoMurm5+tAdYvtHhJ0vy16eoBoKp+UVUPN5sX78M13wFcnORHDPr8P7EP55IkzVObrp6ZZD4/rqqNwMbm8xbgWQu8riRpH7Vu8U9jn7skjahZW/xJHmbmgA9weGcVSZI6NWvwV9Xjh1mIJGk4FtrVI0kaUQa/JPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzyx0rh5pJAwWfRttS5cuXewSdJAx+HXQqup+SqkkQ7mOtD/Z1SNJPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k9Y/BLUs8Y/JLUMwa/JPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k901nwJzksyS1JvpPk+0ne2+xfneRbSX6U5Nokh3ZVgyRpT122+B8BzqyqU4BTgRcneQ7wAeDDVfVUYAdwfoc1SJKm6Sz4a+DnzebjmlcBZwLXN/vXAed2VYMkaU+d9vEnOSTJbcB24Gbgx8CDVfVo85O7geNmOfaCJBNJJiYnJ7ssU5J6pdPgr6pfV9WpwErgWcDJ8zh2bVWNV9X42NhYZzVKUt8MZVRPVT0IfA04DTgmyZLmq5XA1mHUIEka6HJUz1iSY5rPhwNnA3cy+APw6uZna4D1XdUgSdrTkr3/ZMGOBdYlOYTBH5jPVtUNSe4APpPk/cC3gU90WIM0L0mGckxVzfsYaX/pLPir6rvAM2bYv4VBf790wDGQ1Qc+uStJPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9UxG4YGVJJPAXYtdhzSD5cB9i12ENIsnV9Ues1yORPBLB6okE1U1vth1SPNhV48k9YzBL0k9Y/BL+2btYhcgzZd9/JLUM7b4JalnDH5J6hmDX1qAJP+UZHuS2xe7Fmm+DH5pYf4ZePFiFyEthMEvLUBVfQN4YLHrkBbC4JeknjH4JalnDH5J6hmDX5J6xuCXFiDJp4H/BE5KcneS8xe7Jqktp2yQpJ6xxS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQz/we5jRcgfjYKUgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plot_len_hist(stat_filtered_final)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "aEuF0EN-JMiI",
        "outputId": "e3c43d7e-486f-4e49-f514-f8cf2059d968"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEGCAYAAACpXNjrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhcVbnv8e+PQBgNYcjJwYSQgBEuKDK0DIIaiMYgQzgKCChELpd4FJm8IqDeGxzwAUeIV5BIkEGQSYaoQIwhIIoMSQhDgkhOmBLDIGHmCgTe88daDUWnunp3d+3qVPXv8zz19N5rT++uQL+911p7LUUEZmZm9bRaXwdgZmatx8nFzMzqzsnFzMzqzsnFzMzqzsnFzMzqbvW+DmBVsfHGG8fIkSP7Ogwzs6Yyd+7cf0bEkI7lTi7ZyJEjmTNnTl+HYWbWVCQ9Wq3c1WJmZlZ3Ti5mZlZ3Ti5mZlZ3Ti5mZlZ3Ti5mZlZ3Ti5mZlZ3Ti5mZlZ3Ti5mZlZ3Ti5mZlZ3fkO/helbqrk9JnuiODMrh59czMys7pxczMys7pxczMys7pxczMys7pxczMys7pxczMys7pxczMys7rp8z0XSbsCpwGZ5fwEREZuXG5qZmTWrIi9RTgNOAOYCb5QbjpmZtYIiyeX5iLih9EjMzKxldJpcJO2QF2dL+gFwNfBq+/aImFdybGZm1qRqPbn8qMN6W8VyAHvWPxwzM2sFnSaXiNgDQNLmEbG4cpskN+abmVmninRFvqpK2ZX1DsTMzFpHrTaXrYBtgPUlfapi0yBgrbIDMzOz5lWrzWVLYB9gMLBvRfmLwFFlBmVmZs2tVpvLdcB1knaNiL82MCYzM2tyRd5zOVTSIR3Kngfm5ARkZmb2DkUa9NcEtgMeyp9tgeHAkZLO7OwgSedLekrS/RVlG0qaKemh/HODXC5JUyQtknRvxTs2SJqY939I0sSK8h0l3ZePmSJJta5hZmaNUyS5bAvsERE/jYifAh8DtgL+AxhX47gLgPEdyk4GZkXEaGBWXgfYCxidP5OAcyAlCmAysDOwEzC5IlmcQ2r7aT9ufBfXMDOzBimSXDYA1qtYXxfYMCLeoOKN/Y4i4k/A8g7FE4AL8/KFwP4V5RdFcjswWNImwCeAmRGxPCKeBWYC4/O2QRFxe0QEcFGHc1W7hpmZNUiRNpfvA/Ml3UwaEfkjwPckrQv8sZvXGxoRy/LyE8DQvDwMeLxivyW5rFb5kirlta6xEkmTSE9KjBgxopu3YmZmnekyuUTENEnXk6qlAL4eEf/Iyyf29MIREZKip8fX4xoRMRWYCtDW1lZqLGZm/UnRycJWA54GngXeI+kjPbzek7lKi/zzqVy+FNi0Yr/huaxW+fAq5bWuYWZmDdJlcpF0BvAX4BukJ5UTga/28HrTgfYeXxOB6yrKD8+9xnYhDfO/DJgBjJO0QW7IHwfMyNtekLRL7iV2eIdzVbuGmZk1SJE2l/2BLSOi08b7aiT9GhgDbCxpCanX1+nAFZKOBB4FDsq7Xw98ElgEvAIcARARyyV9B7gr7/ftiGjvJPAlUo+0tYEb8oca1zAzswYpklwWA2tQo2dYNRHR8cXLdmOr7BvA0Z2c53zg/Crlc4D3VSl/pto1rHnoW+p0W0x205hZMyiSXF4h9RabxTsnCzu2tKjMzKypFUku0/PHzMyskCJdkS+UtDYwIiIebEBMZmbW5Ir0FtsXmA/cmNe3k+QnGTMz61SR91xOJb1A+RxARMwHPM2xmZl1qkhyeT0inu9Q9mYZwZiZWWso0qC/QNKhwABJo4FjgdvKDcvMzJpZkSeXY4BtSN2Qfw28ABxfZlBmZtbcivQWe4U09Ms3yg/HzMxaQafJRdJvgVojCu9XSkRmZtb0aj25/LBhUZiZWUvpNLlExC2NDMTMzFpH0flczMzMCnNyMTOzunNyMTOzuuuyK7Kk95Jmn9yscv+I2LPEuMzMrIkVeUP/SuDnwC+AN8oNx8zMWkGR5LIiIs4pPRIzM2sZtV6i3DAv/lbSl4BreOdMlMurHmhmZv1erSeXuaQ39NsnND+xYlvgYffNzKwTtV6iHNXIQMzMrHUUmYnyaEmDK9Y3yNVkZmZmVRV5z+WoiHiufSUingWOKi8kMzNrdkWSywBJ7e0uSBoADCwvJDMza3ZFuiLfCFwu6dy8/oVcZmZmVlWR5HISKaF8Ma/PBM4rLSIzM2t6RWaifBM4J3+sCn1LNbfH5E7nXDMza0lFxhZ7mCozUkaE33MxM7OqijTotwEfzJ8PA1OAX/XmopJOkLRA0v2Sfi1pLUmjJN0haZGkyyUNzPuumdcX5e0jK85zSi5/UNInKsrH57JFkk7uTaxmZtZ9XSaXiHim4rM0Is4E9u7pBSUNA44F2iLifcAA4GDgDOAnEfEe4FngyHzIkcCzufwneT8kbZ2P2wYYD5wtaUDuzfYzYC9ga+CQvK+ZmTVIkZcod6j4tEn6T4p1BKhldWBtSasD6wDLgD2Bq/L2C4H98/KEvE7ePjZ3jZ4AXBYRr0bEw8AiYKf8WRQRiyPiNeCyvK+ZmTVIkSTxo4rlFcAjwEE9vWBELJX0Q+Ax4P8DfyCNY/ZcRKzIuy0BhuXlYcDj+dgVkp4HNsrlt1ecuvKYxzuU71wtFkmTgEkAI0aM6OktmZlZB0V6i+1RzwtK2oD0JDEKeI40X8z4el6jqIiYCkwFaGtrc5cuM7M6KVItdpykQUrOkzRP0rheXPNjwMMR8XREvA5cDewGDM7VZADDgaV5eSmwaY5ldWB94JnK8g7HdFZuZmYNUqS32P+MiBeAcaTqqMOA03txzceAXSStk9tOxgILgdnAAXmficB1eXl6XidvvykiIpcfnHuTjQJGA3cCdwGjc++zgaRG/+m9iNfMzLqpSJtL+xuCnwQuiogFlWONdVdE3CHpKmAeqQ3nblLV1O+ByyR9N5dNy4dMAy6WtAhYTkoW5DiuICWmFcDREfEGgKQvAzNIPdHOj4gFPY3XzMy6r0hymSvpD6Q2klMkvQt4szcXjYjJwOQOxYtJPb067vsv4MBOznMacFqV8uuB63sTo5mZ9VyR5HIksB2wOCJekbQRcES5YZmZWTMrOrbYvIr1Z0gN6mZmZlUVadA3MzPrFicXMzOru0LDuOTxuoZW7h8Rj5UVlJmZNbciQ+4fQ+rZ9SRv9xILYNsS4zIzsyZW5MnlOGDL3JBvZmbWpSLJ5XHg+bIDsepqzXLpGS7NbFXVaXKR9JW8uBi4WdLvgVfbt0fEj0uOzczMmlStJ5d35Z+P5c/A/IEq0x6bmZm16zS5RMS3ACQdGBFXVm6TVHU4FjMzMyj2nsspBcvMzMyA2m0ue5FGQh4maUrFpkGkUYjNzMyqqtXm8g9gDrAfaRridi8CJ5QZlJmZNbdabS73APdIuqRibnszM7Mu1aoWuyIiDgLulrRS77CI8Bv6ZmZWVa1qsePyz30aEYiZmbWOWtViy/Lix4A/RcRDjQnJzMyaXZHhX0YA50oaSWrY/xNwa0TMLzEuMzNrYkVmopwMIGlt4CjgROBMYEC5ofUPtcYOMzNrVkWG3P8msBuwHnA38FXg1pLjMjOzJlakWuxTpJcmfw/cAvw1Il6tfYiZmfVnRarFdpA0iPT08nFgqqSnImL30qMzaxJdVW96egTrb4pUi70P+DDwUaCNNL+Lq8XMzKxTRarFTif1EJsC3BURr5cbkpmZNbsi1WJ+idLMzLqlyJD7ZmZm3eLkYmZmddet5CJptdxzrFckDZZ0laS/SXpA0q6SNpQ0U9JD+ecGeV9JmiJpkaR7Je1QcZ6Jef+HJE2sKN9R0n35mCmS/KaimVkDdZlcJF0qaZCkdYH7gYWSTuzldc8CboyIrYAPAA8AJwOzImI0MCuvA+wFjM6fScA5Oa4NgcnAzsBOwOT2hJT3OariuPG9jNfMzLqhyJPL1hHxArA/cAMwCjispxeUtD7wEWAaQES8FhHPAROAC/NuF+brkcsviuR2YLCkTYBPADMjYnlEPAvMBMbnbYMi4vaICOCiinOZmVkDFEkua0hag/QLenruitybN8JGAU8Dv5R0t6Tz8lPR0IqRmJ8AhublYaR3a9otyWW1ypdUKV+JpEmS5kia8/TTT/filszMrFKR5HIu8AiwLvAnSZsBL/TimqsDOwDnRMT2wMu8XQUGQH7iKP2V5oiYGhFtEdE2ZMiQsi9nZtZvFHnPZQrpBcp2j0raoxfXXAIsiYg78vpVpOTypKRNImJZrtp6Km9fCmxacfzwXLYUGNOh/OZcPrzK/tYNHs7EzHqjSIP+UEnTJN2Q17cGJnZxWKci4gngcUlb5qKxwEJgesV5JwLX5eXpwOG519guwPO5+mwGME7SBrkhfxwwI297QdIuuZfY4RXnMjOzBigy/MsFwC+Bb+T1vwOXkxvke+gY4BJJA4HFwBGkRHeFpCOBR4GD8r7XA58EFgGv5H2JiOWSvgPclff7dkQsz8tfynGvTeqEcEMvYjUzs24qklw2jogrJJ0CEBErJL3Rm4vmWSzbqmwaW2XfAI7u5DznA+dXKZ8DvK83MZqZWc8VadB/WdJG5Ab29qqpUqMyM7OmVuTJ5Sukdo8tJP0FGAIcUGpUZmbW1Ir0Fpsn6aPAloCABz3svpmZ1VJksrCjgUsiYkFe30DSIRFxdunRmVmvuEu59ZUibS5H5eFZAMhDrRxVXkhmZtbsiiSXAZWjCksaAAwsLyQzM2t2RRr0bwQul3RuXv9CLjMzM6uqSHI5iZRQvpjXZwLnlRaRmZk1vSK9xd6UdAFwU0Q8WH5IZmbW7Ir0FtsP+AGpnWWUpO1IQ63sV3ZwVq6uehKZmfVUkWqxyaSZHm+GNHSLpFFlBmXFODmY2aqqSG+x1yOi43Av7hxvZmadKvLkskDSoaQuyaOBY4Hbyg3LzMyaWZHkcgxpuP1XgUtJ86h8p8ygzKx/88gCza9Ictk7Ir7B2/O5IOlA4MrSorKmV+uXg38xmLW+Im0upxQsMzMzA2o8uUjaizQD5DBJUyo2DQJWlB2YrdrcU83MaqlVLfYPYA6wHzC3ovxF4IQyg7LW5sRk1vo6TS4RcQ9wj6RLPX+LmZl1R5EG/Z0knQpslvcXaWr7zcsMzMzMmleR5DKNVA02F3ij3HDMzKwVFEkuz0fEDaVHYmY94jYsWxUVSS6zJf0AuJr0IiUAETGvtKhajP/nrx+/XGfWHIokl53zz7aKsgD2rH84ZmbWCorM57JHIwIxM7PW0eUb+pKGSpom6Ya8vrWkI8sPzczMmlWR4V8uIA1W+e68/nfg+LICMjOz5lekzWXjiLhC0ikAEbFCkrskW1PygJpmjVHkyeVlSRuRJwiTtAvQcfKwbpM0QNLdkn6X10dJukPSIkmXSxqYy9fM64vy9pEV5zgllz8o6RMV5eNz2SJJJ/c2VjMz654iTy5fAaYDW0j6CzAEOKAO1z4OeIA0ECbAGcBPIuIyST8HjgTOyT+fjYj3SDo47/cZSVsDBwPbkKrs/ijpvflcPwM+DiwB7pI0PSIW1iFmW8W527fZqqHLJ5f8PstHgQ8BXwC2iYh7e3NRScOBvYHz8rpIXZuvyrtcCOyflyfkdfL2sXn/CcBlEfFqRDwMLAJ2yp9FEbE4Il4DLsv7mplZgxTpLXYgsHZELCD9wr9c0g69vO6ZwNeAN/P6RsBzEdE+lP8SYFheHgY8Dqm9h1Qlt1FleYdjOitfiaRJkuZImvP000/38pbMzKxdkTaX/xMRL0raHRhLGmvsnJ5eUNI+wFMRMbfLnUsWEVMjoi0i2oYMGdLX4ZiZtYwiyaW9Z9jewC8i4vfAwF5cczdgP0mPkKqs9gTOAgZLam8DGg4szctLgU0B8vb1gWcqyzsc01m5mZk1SJEG/aWSziU1kJ8haU2KJaWqIuIU8jTJksYAX42Iz0q6ktRR4DJgInBdPmR6Xv9r3n5TRISk6cClkn5MatAfDdxJmhJgtKRRpKRyMHBoT+M1MyvKXd3fViS5HASMB34YEc9J2gQ4sYRYTgIuk/Rd4G5S9Rv558WSFgHLScmCiFgg6QpgIWna5aMj4g0ASV8mvfg5ADg/txeZmVmDFBlb7BXSiMjt68uAZfW4eETcDNyclxeTenp13OdfwIGdHH8acFqV8uuB6+sRo/UfHnHZrH56XL1lZmbWmU6TS25bMTMz67Za1WJ/BXaQdHFEHNaogMxWVX7736y4WslloKRDgQ9J+lTHjRFxdZVjzKwKt+dYf1Mrufwn8FlgMLBvh21BRSO/mZlZpU6TS0T8GfizpDkRMa2z/czMzDoq8p7LxZKOBT6S128Bfh4Rr5cXlpmZNbMiyeVsYI38E+Aw0thi/6usoMzMrLkVSS4fjIgPVKzfJOmesgIyM7PmV2jgSklbtK9I2py3B7M0MzNbSZEnlxOB2ZIWkwaF3Aw4otSozMysqRUZW2yWpNHAlrnowYh4tdywzMysXvpitOYiTy7kZNKrqY3NzKz/KJRczKzveNgZa0ZOLmarACcQazVd9haTNKtImZmZWbtOn1wkrQWsA2wsaQNSTzGAQcCwBsRmZmZNqla12BeA40nz08/l7eTyAvD/So7LzMyaWK2BK88CzpJ0TET8tIExmVmDeCoAK0uR91x+KulDwMjK/SPiohLjMjPrVF+8t2Hd02VykXQxsAUwn7eHfQnAycWsxZX1S9y941pfka7IbcDWEeE/B8zsLa5Ss1qKJJf7gX8HlpUci5m1ED+d9G9FksvGwEJJdwJvjSkWEfuVFpWZmTW1Isnl1LKDMDOz1lKkt9gtjQjEzMxaR5HeYi+SeocBDCRNefxyRAwqMzAzszK4I0JjFHlyeVf7siQBE4BdygzKzKyn3JFg1VBkmuO3RHIt8ImeXlDSppJmS1ooaYGk43L5hpJmSnoo/9wgl0vSFEmLJN0raYeKc03M+z8kaWJF+Y6S7svHTMlJ0czMGqRItdinKlZXI7338q9eXHMF8L8jYp6kdwFzJc0EPg/MiojTJZ0MnAycBOwFjM6fnYFzgJ0lbQhMzvFEPs/0iHg273MUcAdwPTAeuKEXMZtZP+G3/+ujSG+xfSuWVwCPkKrGeiQilpHfmYmIFyU9QBpleQIwJu92IXAzKblMAC7KL3HeLmmwpE3yvjMjYjlATlDjJd0MDIqI23P5RcD+OLmYWS+5yq24Im0uR5R1cUkjge1JTxhDc+IBeAIYmpeHAY9XHLYkl9UqX1KlvNr1JwGTAEaMGNHzGzEzs3coMlnYcEnXSHoqf34jaXhvLyxpPeA3wPER8ULltvyUUvrzZ0RMjYi2iGgbMmRI2ZczM+s3ijTo/xKYTprX5d3Ab3NZj0lag5RYLomIq3Pxk7m6i/zzqVy+FNi04vDhuaxW+fAq5WZm1iBF2lyGRERlMrlA0vE9vWDuuTUNeCAiflyxaTowETg9/7yuovzLki4jNeg/HxHLJM0AvtfeqwwYB5wSEcslvSBpF1J12+GA56Mxsz7V2/aaZutMUCS5PCPpc8Cv8/ohwDO9uOZuwGHAfZLm57Kvk5LKFZKOBB4FDsrbrgc+CSwCXgGOAMhJ5DvAXXm/b7c37gNfAi4A1iY15Lsx38yaWrP1YlNXI+lL2oz0l/+upHaQ24BjI+Kx8sNrnLa2tpgzZ06PjnUPEjNrVr1NTJLmRkRbx/IivcUeBTwCspmZFVbkJcpRwDGsPM2xE46ZmVVVpM3lWlID/G+BN8sNx8zMWkGR5PKviJhSeiRmZtYyiiSXsyRNBv7AO2einFdaVGZm1tSKJJf3k7oO78nb1WKR183MzFZSJLkcCGweEa+VHYyZmbWGIsO/3A8MLjsQMzNrHUWeXAYDf5N0F+9sc3FXZDMzq6pIcplcehRmZtZSiryhf0vluqTdSeOL3VL9CDMz6++KPLkgaXvgUFLj/sOk4fLNzMyq6jS5SHov6QnlEOCfwOWkgS73aFBsZmbWpGo9ufwNuBXYJyIWAUg6oSFRmZlZU6vVFflTwDJgtqRfSBoLeGx5MzPrUqfJJSKujYiDga2A2cDxwL9JOkfSuEYFaGZmzafLlygj4uWIuDQi9iXNR383cFLpkZmZWdMq8ob+WyLi2YiYGhFjywrIzMyaX7eSi5mZWRFOLmZmVndOLmZmVndOLmZmVndOLmZmVndOLmZmVndOLmZmVndOLmZmVndOLmZmVndOLmZmVnctm1wkjZf0oKRFkk7u63jMzPqTlkwukgYAPwP2ArYGDpG0dd9GZWbWf7RkcgF2AhZFxOKIeA24DJjQxzGZmfUbtWaibGbDgMcr1pcAO3fcSdIkYFJefUnSgyXEsjFpmuj+qr/fP/g7AH8HsIp+Bzq113NAblatsFWTSyERMRWYWuY1JM2JiLYyr7Eq6+/3D/4OwN8B9L/voFWrxZYCm1asD89lZmbWAK2aXO4CRksaJWkgcDAwvY9jMjPrN1qyWiwiVkj6MjADGACcHxEL+iicUqvdmkB/v3/wdwD+DqCffQeKiL6OwczMWkyrVouZmVkfcnIxM7O6c3KpE0lrSbpT0j2SFkj6Vi4fJemOPAzN5bmDQcuSNEDS3ZJ+l9f71f0DSHpE0n2S5kuak8s2lDRT0kP55wZ9HWdZJA2WdJWkv0l6QNKu/ez+t8z/9u2fFyQd35++A3ByqadXgT0j4gPAdsB4SbsAZwA/iYj3AM8CR/ZhjI1wHPBAxXp/u/92e0TEdhXvNZwMzIqI0cCsvN6qzgJujIitgA+Q/nvoN/cfEQ/mf/vtgB2BV4Br6EffATi51E0kL+XVNfIngD2Bq3L5hcD+fRBeQ0gaDuwNnJfXRT+6/y5MIN0/tPD3IGl94CPANICIeC0inqOf3H8VY4H/iohH6WffgZNLHeUqofnAU8BM4L+A5yJiRd5lCWlomlZ1JvA14M28vhH96/7bBfAHSXPzEEMAQyNiWV5+AhjaN6GVbhTwNPDLXD16nqR16T/339HBwK/zcr/6Dpxc6igi3siPwsNJg2du1cchNYykfYCnImJuX8eyCtg9InYgjcp9tKSPVG6M1P+/Vd8BWB3YATgnIrYHXqZD9U+L3/9bcvvifsCVHbf1h+/AyaUEuRpgNrArMFhS+8uqrTwMzW7AfpIeIY1CvSep7r2/3P9bImJp/vkUqa59J+BJSZsA5J9P9V2EpVoCLImIO/L6VaRk01/uv9JewLyIeDKv96vvwMmlTiQNkTQ4L68NfJzUkDkbOCDvNhG4rm8iLFdEnBIRwyNiJKkq4KaI+Cz95P7bSVpX0rval4FxwP2k4Ycm5t1a9nuIiCeAxyVtmYvGAgvpJ/ffwSG8XSUG/ew78Bv6dSJpW1Ij3QBS0r4iIr4taXPSX/IbAncDn4uIV/su0vJJGgN8NSL26W/3n+/3mry6OnBpRJwmaSPgCmAE8ChwUEQs76MwSyVpO1KnjoHAYuAI8v8T9IP7h7f+sHgM2Dwins9l/ea/AXByMTOzErhazMzM6s7JxczM6s7JxczM6s7JxczM6s7JxczM6s7JxRpO0ktd79Wr8x8vaZ16XE/SmpL+mEe3/Ux9IlzpGl8v47z1IGlM+wjXDbreSEmHNup6Vh4nF2tFxwPrdLlXMdsD5FFuL6/TOTtaZZNLb1WMzlDUSKBbyaUH17AGcHKxVYKkLSTdmAd7vFXSVrn8AklTJN0mabGkA3L5apLOznOGzJR0vaQDJB0LvBuYLWl2xflPy3Pt3C5ppQED81wb10q6N++zraR/A34FfDA/uWzR4ZhjJS3Mx1yWy9aVdL7S3D53S5qQyz8v6ep8jw9J+n4uPx1YO5//klz2uXz8fEnnShqQy1+qdh+Shkq6JpffI+lDtc5T49/gPfkp7R5J8yrudz29PT/LJXm0ayT9X0l3Sbpf0tSK8pslnak0l81xknaUdEv+t51RMQRKteudDnw4x3yC0mCwP8jXuVfSF/KxY/J/J9OBhfl7/30+1/1lPWVaN0SEP/409AO8VKVsFjA6L+9MGj4G4ALSwH+rAVsDi3L5AcD1ufzfSXPFHJC3PQJsXHHuAPbNy98Hvlnl+j8FJuflPYH5eXkM8LtO7uMfwJp5eXD++T3SKAQAg4G/A+sCnye9rb4+sBbpDe1NO34fwP8AfguskdfPBg6vdR/A5cDxeXlAvkan56nx73IH8B95eS3S098Y4HnSuHCrAX8lDcwJsGHFsRdXxHYzcHZeXgO4DRiS1z8DnN/F9X5Xcd5JFfe5JjCHNPLyGNKgmKPytk8Dv6g4bv2+/u+8v3/8OGl9TtJ6wIeAK/Mfv5B+kbS7NiLeJP2F2v7UsTtwZS5/ovIppYrXgPZ2g7mkcd862p30C4qIuEnSRpIGdRH6vcAlkq4Frs1l40gDeH41r69FGu4D0kRR7UOBLAQ2Ax7vcM6xpAmm7srfxdq8PcBhZ/exJ3B4jv0N4HlJh9U4z0qUxkMbFhHX5PP8K5cD3BkRS/L6fFLV1Z+BPSR9jZQUNgQWkBIapIQHsCXwPmBmPtcAYFkX16s0Dti2/YmVlDhH5+/izoh4OJffB/xI0hmk5HRrZ/dqjeHkYquC1UjzvmzXyfbKschW+u1TwOuR/5wF3qB+/93vTZoYa1/gG5Len+P7dEQ8WLmjpJ155310FoeACyPilCrbunMftc7TXSvFLWkt0tNQW0Q8LulUUiJt93JFHAsiYtd3BJcH9yxAwDERMaPD8WMqrkFE/F3SDsAnge9KmhUR3y54DSuB21ysz0XEC8DDkg6ENIOlpA90cdhfgE8rtb0MJVWTtHsRKPrLq92twGfz9ccA/8xxVSVpNVK11mzgJNJf1OsBM4BjKtofti9w7dclrZGXZwEHKLX3tLcFbdbF8bOAL+b9ByjNBtnpeSRdJGmnyhNExIvAEkn7533WVEWPuyraE8k/85PnAZ3s9yAwRNKu+bxrSNqmxvU6/tvNAL7Y/v1Ieq/SoJDvIOndwCsR8SvgB6Rh/q0POblYX1hH0pKKz1dIv9iPlHQPqXplQhfn+A1p7pCFpEb3eaS2AYCpwI1dVJV1dCqwo6R7SY3KE2vvzgDgV5LuI432PCXSPD7fIbUz3CtpQV7vynG16SkAAADOSURBVNS8/yURsRD4Jmkmy3tJM5pu0sXxx5GqqO4jVZdt3cV5tiW1F3V0GHBs3v82UltWVflef0GaTmAGcFcn+71GSjxn5H/b+aQq0M6udy/wRm6YP4E0uvJCYJ6k+4Fzqf7E9n7gzlxtNxn4bmexW2N4VGRrWpLWi4iXlIYyvxPYLdJ8ItaJ3I40LSIO7OtYrLU5uVjTknQzqUfWQOD7EXFBnwZkZm9xcjEzs7pzm4uZmdWdk4uZmdWdk4uZmdWdk4uZmdWdk4uZmdXdfwMv/7EXzPziOAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's try to remove all pairs contains digits"
      ],
      "metadata": {
        "id": "mES_tDWwq4g5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "digits_in_sent = {idx for lang in ['rus', 'eng']\n",
        "                  for idx, data in enumerate(filtered_final[lang]) \n",
        "                  if any(char.isdigit() for char in data)}\n",
        "\n",
        "print(f'Count of pairs with digits: {len(digits_in_sent)}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "svE2mTNiqHgv",
        "outputId": "49e957aa-cac2-4364-c68a-db3e47151ed3"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Count of pairs with digits: 872012\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "filtered_small = {}\n",
        "for key, data in filtered_final.items(): \n",
        "    filtered_small[key] = [sent for idx, sent in enumerate(data) \n",
        "                                    if idx not in digits_in_sent]\n",
        "\n",
        "print(f'{len(digits_in_sent)} elements was removed')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_E-Bqxzuq_Ue",
        "outputId": "a1d8e6ea-7930-4bba-ba22-d2ff31ebd122"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "872012 elements was removed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stat_filtered_small = get_statistics(filtered_small['rus'], \n",
        "                                     filtered_small['eng'])\n",
        "print(stat_filtered_small[0])\n",
        "plot_counts_boxplot(stat_filtered_small)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "iSr1LDXcrQz4",
        "outputId": "427d59e1-9614-4e4f-8ca8-a63754e5756c"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Lenght statistics:\n",
            "Rus:{'mean': 50.46132072167631, 'argmin': 40, 'argmax': 170, 'min': 29, 'max': 76}\n",
            "Eng{'mean': 51.42139937358657, 'argmin': 40, 'argmax': 96, 'min': 29, 'max': 76}\n",
            "Rus / Eng {'mean': 1.0039682494935338, 'argmin': 601, 'argmax': 2710, 'min': 0.5, 'max': 2.0}\n",
            "Pairs\n",
            "Smallest: ('There are two possibilities:\\n', 'Для этого есть два варианта:\\n')\n",
            "Longest: ('You will find the booking class on your ticket: where is the booking class?\\n', 'Класс бронирования указан в Вашем билете: где указан класс бронирования?\\n')\n",
            "Smallest ratio: ('Below is the tungsten alloy radiation shielding terms and definition.\\n', 'Радиационные термины и определения\\n')\n",
            "Biggest ratio: ('What is your position in this regard?\\n', 'Что говорит Библия в этом смысле, и как мне следует поступать в дальнейшем?\\n')\n",
            "Counts of pairs: 799472\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAP+klEQVR4nO3df4xlZX3H8fenrIQfFVnccUtZcVdEaNMI2okKGKMoxvoDEa2xsc3WEtHUUAy1ikYDWv9Ak2qJTayL1i7EKoI/VqGh4la0Jq10FlERNCgVZQPsAMsPtQHRb/+4Z8u4OzN7ZnafOzuc9yu5ufece358Schnnn3uc54nVYUkaTh+a6kLkCSNl8EvSQNj8EvSwBj8kjQwBr8kDcyKpS6gj1WrVtXatWuXugxJWla2bNlyV1VN7Lx/WQT/2rVrmZqaWuoyJGlZSXLrbPvt6pGkgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYAx+SRoYg1+SBmZZPMAljUuSsdzHdTC0lAx+aYaFBnISQ1zLjl09kjQwBr8kDYzBL0kDY/BL0sAY/JI0MAa/JA2MwS9JA2PwS9LAGPySNDAGvyQNjMEvSQNj8EvSwBj8kjQwBr8kDYzBL0kDY/BL0sAY/JI0MAa/JA2MwS9JA2PwS9LANAv+JMckuX7G6/4kb0lyWJKrk9zcva9sVYMkaVfNgr+qflBVx1fV8cAfAr8APg+cC2yuqqOBzd22JGlMxtXV8wLgR1V1K/AKYGO3fyNw2phqkCQxvuB/LfCp7vPqqrq9+3wHsHq2E5KcmWQqydT09PQ4apSkQWge/En2B04FLtv5u6oqoGY7r6o2VNVkVU1OTEw0rlKShmMcLf4/Aq6rqju77TuTHA7QvW8bQw2SpM44gv9PeKSbB+CLwPru83pg0xhqkCR1mgZ/koOBU4DPzdh9AXBKkpuBF3bbkqQxWdHy4lX1c+DxO+27m9EoH0nSEvDJXUkaGINfkgbG4JekgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYAx+SRoYg1+SBsbgl6SB2W3wJzkoybuTXNRtH53kZe1LkyS10KfF/wngQeCEbnsr8L5mFUmSmuoT/EdV1QeAXwJU1S+ANK1KktRMn+B/KMmBdEskJjmK0b8AJEnLUJ/5+M8DrgKemOSTwEnAn7csSpLUzm6Dv6quTnId8GxGXTxnV9VdzSuTJDXRZ1TPK4GHq+rKqroCeDjJae1LkyS10KeP/7yqum/HRlXdy6j7R5K0DPUJ/tmOabpWrySpnT7BP5Xkg0mO6l4fBLa0LkyS1Eaf4D8LeAi4tHs9CLy5ZVGSpHb6jOr5OXDuGGqRJI3BboM/yVOBtwJrZx5fVSe3K0uS1EqfH2kvA/4R+Bjwq7blSJJa6xP8D1fVR5pXIkkaiz4/7n4pyV8mOTzJYTtezSuTJDXRp8W/vnv/mxn7Cnjy3i9HktRan1E968ZRiLS3HXbYYWzfvr35fZK2s5SvXLmSe+65p+k9NCx9RvUcBJwDHFlVZyY5Gjimm7dH2mdt376dqlrqMvZY6z8sGp6+K3A9BJzYbbsClyQtY67AJUkD4wpckjQwfYL/fH5zBa7NwNv7XDzJoUkuT/L9JDclOaEbDnp1kpu795WLL1+StFC7Df6q+jJwOqPlFj8FTFbVV3te/0Lgqqo6FjgOuInRvD+bq+poRn9EnAdIksaozwpcm6vq7h0rcFXVXUk29zjvccBzgY8DVNVD3SIurwA2dodtBFzNS5LGaM7hnEkOAA4CVnXdMTt+0D0EOKLHtdcB08AnkhzHaA7/s4HVVXV7d8wdwOpF1i5JWoT5WvxvZBTWx3bvO16bgH/oce0VwDOAj1TV04Fdpneu0SDrWQdaJzkzyVSSqenp6R63kyT1MWfwV9WF3VO7b62qJ1fVuu51XFX1Cf7bgNuq6pvd9uWM/hDcmeRwgO592xz331BVk1U1OTExsaD/KEnS3PpM2fDhJCey63z8F+/mvDuS/DTJMVX1A+AFwI3daz1wQfe+afHlS5IWqs+UDZcARwHX88h8/AXMG/yds4BPJtkfuAV4PaN/ZXwmyRnArcBrFlG3JGmR+szOOQn8fi1i0pOqur47f2cvWOi1JEl7R58HuG4Afqd1IZKk8ejT4l8F3JjkWmZM1VBVpzarSpLUTJ/gP791EZKk8ekzqudrSZ4EHF1VX+nm59+vfWmSpBb6TNnwBkZj8D/a7ToC+ELLoiRJ7fT5cffNwEnA/QBVdTPwhJZFSZLa6RP8D1bVQzs2kqxgjmkWJEn7vj7B/7Uk7wQOTHIKcBnwpbZlSZJa6RP85zKaZfO7jCZu+1fgXS2LkiS102dUz6+Bi4CLkhwGrFnMU7ySpH1Dn1E91yQ5pAv9LYz+AHyofWmSpBb6dPU8rqruZ7T84sVV9Syca0eSlq0+wb+imzf/NcAVjeuRJDXWJ/jfC/wb8MOq+u8kTwZubluWJKmVPj/uXsZoCOeO7VuAV7UsSpLUTp8WvyTpUcTgl6SBMfglaWD6rLl7ziy77wO2dEsrSpKWkT4t/kngTYymYz6C0bQNL2b0INfbGtYmSWqgzwpca4BnVNXPAJKcB1wJPJfRk7wfaFeeJGlv69PifwIz1toFfgmsrqr/3Wm/JGkZ6NPi/yTwzSSbuu2XA/+S5GDgxmaVSZKa6PMA198muQo4sdv1pqqa6j6/rlllkqQm+rT4Aa4Dtu44PsmRVfWTZlVJkprpM5zzLOA84E7gV0AYLb34tLalSZJa6NPiPxs4pqrubl2MJKm9PqN6fsrogS1J0qNAnxb/LcA1Sa5kxvDNqvpgs6okSc30Cf6fdK/9u5ckaRnrM5zzPTvvS9J3NJAkaR8zZx9/km/M+HzJTl9f26wiSVJT87XcD57x+Q92+i4NapH2qjrvEDj/cUtdxh6r8w5Z6hL0KDNf8Nccn2fbnlWSHwMPMBr//3BVTSY5DLgUWAv8GHhNVW3vWa/UW95zP1W9/lfdpyWhzl/qKvRoMl/wH5rklYy6gw5Ncnq3P8BCmlHPr6q7ZmyfC2yuqguSnNttv30hRUuSFm++4P8acOqMzy+f8d3X9+CerwCe133eCFyDwS9JYzNn8FfV6/fC9Qv4cpICPlpVGxhN6Xx79/0dwOq9cB9JUk+th2U+p6q2JnkCcHWS78/8sqqq+6OwiyRnAmcCHHnkkY3LlKThaLrYelVt7d63AZ8HngncmeRwgO592xznbqiqyaqanJiYaFmmJA3KfOP4/7h7X7eYCyc5OMljd3wGXgTcAHwRWN8dth7YNPsVJEktzNfif0f3/tlFXns18I0k32b0wNeVVXUVcAFwSpKbgRd225KkMZmvj//uJF8G1iX54s5fVtWps5wz8/tbgONm2X838IKFFipJ2jvmC/6XAs8ALgH+bjzlSJJam28450PAfyU5saqmk/x2t/9nY6tOkrTX9RnVszrJt4DvATcm2ZJk57l7JEnLRJ/g3wCcU1VPqqojgb/u9kmSlqE+wX9wVX11x0ZVXcNvztwpSVpGei29mOTdjH7kBfhTRssxSpKWoT4t/r8AJoDPMRrTv6rbJ0lahvosvbgd+Ksx1CJJGoOmc/VIkvY9Br8kDYzBL0kDs9s+/iQTwBsYrZH7/8dXlT/wStIy1Gc45ybgP4CvMFo0XZK0jPUJ/oOqyjVxJelRok8f/xVJXtK8EknSWMzZ4k/yAKPF0gO8M8mDwC+77aqqQ8ZToiRpb5pvWubHjrMQSdJ47LarJ8nmPvskScvDfF09BzCahXNVkpWMungADgGOGENtkqQG5hvV80bgLcDvAtfN2H8/8A8ti5IktTNfH/+FwIVJzqqqD4+xJklSQ33G8W9NcvpO++4DvltV2xrUJElqqE/wnwGcAOxYhet5wBZgXZL3VtUlc50oSdr39An+xwC/V1V3AiRZDVwMPAv4Oo+szCVJWgb6PLm7Zkfod7YBT6yqexg90CVJWkb6tPivSXIFcFm3/apu38HAvc0qkyQ10Sf438wo7E/qti8GPltVBTy/VWGSpDb6rLlbwOXdS5K0zPWZsuH0JDcnuS/J/UkeSHL/OIqTJO19fbp6PgC8vKpual2MJKm9PqN67jT0JenRo0+LfyrJpcAXgAd37KyqzzWrSpLUTJ/gPwT4BfCiGfsKMPi1z0uy+4P2cStXrlzqEvQo02dUz+v35AZJ9gOmgK1V9bIk64BPA49nNPXDn1XVQ3tyD2k2owFpbSUZy32kvanPqJ6nJtmc5IZu+2lJ3rWAe5wNzPyN4P3Ah6rqKcB2RnMBSZLGpM+PuxcB76CbnqGqvgO8ts/Fk6wBXgp8rNsOcDKPPBOwEThtYSVLkvZEn+A/qKqu3Wnfwz2v//fA24Bfd9uPB+6tqh3n34areUnSWPUJ/ruSHMXoB12SvBq4fXcnJXkZsK2qtiymsCRnJplKMjU9Pb2YS0iSZtF3rp4NwLFJtgL/A7yux3knAacmeQlwAKPRQRcChyZZ0bX61wBbZzu5qjZ092VyctJfzyRpL9lti7+qbqmqFwITwLFV9RzglT3Oe0dVramqtYx+E/j3qnodowVdXt0dth7YtNjiJUkL16erB4Cq+nlVPdBtnrMH93w7cE6SHzLq8//4HlxLkrRAfbp6ZrOgp2Kq6hrgmu7zLcAzF3lfSdIe6t3i34l97pK0TM3Z4k/yALMHfIADm1UkSWpqzuCvqseOsxBJ0ngstqtHkrRMGfySNDAGvyQNjMEvSQNj8EvSwBj8kjQwBr8kDYzBL0kDY/BL0sAY/JI0MAa/JA2MwS9JA2PwS9LAGPySNDAGvyQNjMEvSQNj8EvSwBj8kjQwBr8kDYzBL0kDY/BL0sAY/JI0MAa/JA2MwS9JA2PwS9LAGPySNDAGvyQNjMEvSQNj8EvSwBj8kjQwBr8kDUyz4E9yQJJrk3w7yfeSvKfbvy7JN5P8MMmlSfZvVYMkaVctW/wPAidX1XHA8cCLkzwbeD/woap6CrAdOKNhDZKknTQL/hr5Wbf5mO5VwMnA5d3+jcBprWqQJO2qaR9/kv2SXA9sA64GfgTcW1UPd4fcBhwxx7lnJplKMjU9Pd2yTEkalKbBX1W/qqrjgTXAM4FjF3DuhqqarKrJiYmJZjVK0tCMZVRPVd0LfBU4ATg0yYruqzXA1nHUIEkaaTmqZyLJod3nA4FTgJsY/QF4dXfYemBTqxokSbtasftDFu1wYGOS/Rj9gflMVV2R5Ebg00neB3wL+HjDGqQFSTKWc6pqwedIe0uz4K+q7wBPn2X/LYz6+6V9joGsIfDJXUkaGINfkgbG4JekgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYLIcHlhJMg3cutR1SLNYBdy11EVIc3hSVe0yy+WyCH5pX5Vkqqoml7oOaSHs6pGkgTH4JWlgDH5pz2xY6gKkhbKPX5IGxha/JA2MwS9JA2PwS4uQ5J+SbEtyw1LXIi2UwS8tzj8DL17qIqTFMPilRaiqrwP3LHUd0mIY/JI0MAa/JA2MwS9JA2PwS9LAGPzSIiT5FPCfwDFJbktyxlLXJPXllA2SNDC2+CVpYAx+SRoYg1+SBsbgl6SBMfglaWAMfkkaGINfkgbm/wAsZxeja3DXUAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plot_len_hist(stat_filtered_small)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "cKJKSGKnrrgL",
        "outputId": "f3ca7b9a-0606-436a-958e-f694fca037ce"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEGCAYAAACkQqisAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xV1X338c9XFLUag+iUKoigEn2INahT0WgbxScEjYpNMVETpamPpInXpLVqkufBJNqXJs1FU2MkQsRUg2i8kAQlBPHWROUiomCMFLVA8RJBvL2CAX/PH2uNHIaZM3sOc87MmfN9v17nNXv/9m3tg85v1l5rr6WIwMzMrBLbdHcBzMysfjmJmJlZxZxEzMysYk4iZmZWMScRMzOr2LbdXYBa23333WPIkCHdXQwzs7qyYMGCP0REU+t41ZOIpD7AfGBVRJwgaSgwDdgNWACcERHvSNoeuAk4FHgV+FREPJ/PcSlwFrAROD8iZuX4GOBqoA9wQ0Rc2VF5hgwZwvz587v4Ls3MejdJL7QVr8XjrAuAp0vWrwK+GxH7AWtJyYH8c22Ofzfvh6ThwKnAB4ExwA8k9cnJ6VrgOGA4cFre18zMaqSqSUTSIODjwA15XcAo4Pa8y1Tg5Lw8Nq+Ttx+b9x8LTIuI9RHxHLAMOCx/lkXE8oh4h1S7GVvN+zEzs81VuybyPeBfgHfz+m7AaxGxIa+vBAbm5YHACoC8fV3e/714q2Pai29B0gRJ8yXNf+WVV7b2nszMLKtaEpF0AvByRCyo1jWKiohJEdEcEc1NTVu0C5mZWYWq2bB+JHCSpOOBHYBdSI3g/SRtm2sbg4BVef9VwF7ASknbAu8nNbC3xFuUHtNe3MzMaqBqNZGIuDQiBkXEEFLD+H0R8WlgLjAu7zYeuDsvz8jr5O33RRodcgZwqqTtc8+uYcBjwDxgmKShkvrma8yo1v2YmdmWuuM9kYuBaZIuBx4HJuf4ZOAnkpYBa0hJgYhYImk6sBTYAJwTERsBJJ0LzCJ18Z0SEUtqeidmZg1OjTYUfHNzc/g9ETOzzpG0ICKaW8c97ImZmVWs4YY9sZ5DX1O722Jiz6whlysz9Nxym1WLayJmZlYxJxEzM6uYk4iZmVXMScTMzCrmJGJmZhVzEjEzs4o5iZiZWcU6fE9E0pHAZcDeeX8BERH7VLdoZmbW0xV52XAy8EXSVLYbq1scM6uEX4K07lIkiayLiHuqXhIzM6s77SYRSYfkxbmSvgXcAaxv2R4RC6tcNjMz6+HK1US+3Wq9dPTGIM2VbmZmDazdJBIRxwBI2icilpduk+RGdTMzK9TF9/Y2Yrd1dUHMzKz+lGsTOQD4IPB+SZ8o2bQLac50MzNrcOXaRPYHTgD6ASeWxN8Azq5moczMrD6UaxO5G7hb0hER8dvOnljSDsCDwPb5OrdHxERJNwIfAdblXf8+IhZJEnA1cDzwdo4vzOcaD3w17395REzN8UOBG4EdgZnABdFo8/2amXWjIu+JnC7ptFaxdcD8nGjasx4YFRFvStoOeFhSy/smF0VE67aW44Bh+TMSuA4YKak/MJHUOyyABZJmRMTavM/ZwKOkJDIG8DstZmY1UqRhfXtgBPBs/hwEDALOkvS99g6K5M28ul3+lKsljAVuysc9AvSTtAfwMWB2RKzJiWM2MCZv2yUiHsm1j5uAkwvcj5mZdZEiNZGDgCMjYiOApOuAh4CjgCfLHSipD2m4lP2AayPiUUmfB66Q9P+AOcAlEbEeGAisKDl8ZY6Vi69sI95WOSYAEwAGDx5c4Jbri4e8MLPuUqQmsiuwc8n6TkD/nFTWt31IEhEbI2IEqeZymKQDgUuBA4C/AvoDF1dS8M6IiEkR0RwRzU1NTdW+nJlZwyiSRL4JLJL049wo/jjwLUk7Ab8ucpGIeA2YC4yJiNX5kdV64MfAYXm3VcBeJYcNyrFy8UFtxM3MrEY6TCIRMRn4MHAXcCdwVETcEBFvRcRF7R0nqUlSv7y8I/BR4He5LYPcG+tk4Kl8yAzgTCWHkwZ+XA3MAkZL2lXSrsBoYFbe9rqkw/O5zgTKNfSbmVkXK9ImAinZvJL330/SfhHxYAfH7AFMze0i2wDTI+IXku6T1ESal2QR8I95/5mk7r3LSF18PwsQEWskfQOYl/f7ekSsyctfYFMX33twzywzs5oqMinVVcCngCXAuzkcpHdA2hURi4GD24i3OXBj7mF1TjvbpgBT2ojPBw4sVw4zM6ueIjWRk4H9cxuGmZnZe4o0rC8nveNhZma2mSI1kbdJvbPmsPmkVOdXrVRmZlYXiiSRGfljZtal/KJs/eswiUTE1NxFd3BEPFODMpmZWZ3osE1E0omkrrj35vURklwzMTOzQg3rl5HeKn8NICIWAZ4e18zMCiWRP0XEulaxd9vc08zMGkqRhvUlkk4H+kgaBpwP/Ka6xTIzs3pQpCZyHmmu9fXAT4HXgQurWSgzM6sPRXpnvQ18JX/MzMze024SkfRzysxEGBEnVaVEZmZWN8rVRP6tZqUwM7O61G4SiYgHalkQMzOrP0Ua1s3MzNrkJGJmZhVzEjEzs4oVmdnwA8BFwN6l+7c3Q6GZmTWOIjWR24CFwFdJyaTlU5akHSQ9JukJSUskfS3Hh0p6VNIySbdK6pvj2+f1ZXn7kJJzXZrjz0j6WEl8TI4tk3RJZ27czMy2XpFhTzZExHUVnHs9MCoi3pS0HfCwpHuALwHfjYhpkn4InAVcl3+ujYj9JJ0KXAV8StJw4FTSW/N7Ar/OtSOAa4GPAiuBeZJmRMTSCspqZmYVaLcmIqm/pP7AzyV9QdIeLbEcLyuSN/PqdvkTwCjg9hyfSprDHWBsXidvP1aScnxaRKyPiOeAZaRRhQ8DlkXE8oh4B5iW9zUzsxopVxNZQPql3zL1WOkjrKDAcPCS+uTz7EeqNfwX8FpEbMi7rAQG5uWBwAqAiNggaR2wW44/UnLa0mNWtIqPbKccE4AJAIMHD+6o2GZmVlC5lw2Hbu3JI2IjMEJSP+BO4ICtPWeF5ZgETAJobm72fJtmZl2kyMyG5+Qk0LK+q6QvdOYiEfEaMBc4AugnqSV5DQJW5eVVwF75GtsC7wdeLY23Oqa9uJmZ1UiR3lln5yQAQESsBc7u6CBJTS3JJ8/R/lHgaVIyGZd3Gw/cnZdn5HXy9vsiInL81Nx7aygwDHgMmAcMy729+pIa3z1tr5lZDRXpndVHkvIv9JZ2jr4FjtsDmJr33waYHhG/kLQUmCbpcuBxYHLefzLwE0nLgDWkpEBELJE0HVgKbADOyY/JkHQuMAvoA0yJiCWF7trMzLpEkSRyL3CrpOvz+udyrKyIWAwc3EZ8OalnVev4H4FT2jnXFcAVbcRnAjM7KouZmVVHkSRyMSlxfD6vzwZuqFqJzMysbhSZ2fBd0suAlbxwaGZmvViRsbOeo40ZDiOiw/dEzMysdyvyOKu5ZHkHUrtFh2+sm5lZ79dhF9+IeLXksyoivgd8vAZlMzOzHq7I46xDSla3IdVMitRgzMyslyuSDL5dsrwBeB74ZFVKY2ZmdaVI76xjalEQMzOrP0XGzrpA0i5KbpC0UNLoWhTOzMx6tiJjZ/1DRLwOjCYNzX4GcGVVS2VmZnWhSJtIy3wixwM35bGsVO4AM7PeTF9r/1dgTGys2SaK1EQWSPoVKYnMkvQ+4N3qFsvMzOpBkZrIWcAIYHlEvC1pN+Cz1S2WmZnVg6JjZy0sWX+VNFmUmZk1uCKPs8zMzNrkN887oVxjGjReg5qZWaEkkmcnHFC6f0T8d7UKZWZm9aHIy4bnAS+RJqP6Zf78osBxe0maK2mppCWSLsjxyyStkrQof44vOeZSScskPSPpYyXxMTm2TNIlJfGhkh7N8VvzXOtmZlYjRWoiFwD75wb1ztgA/FNELMzdghdImp23fTci/q10Z0nDSfOqfxDYE/i1pA/kzdcCHwVWAvMkzYiIpcBV+VzTJP2Q1JPMk2eZmdVIkYb1FcC6zp44IlZHxMK8/AbwNDCwzCFjgWkRsT4ingOWkeZiPwxYFhHLI+IdYBowNr/wOAq4PR8/FTi5s+U0M7PKtVsTkfSlvLgcuF/SL4H1Ldsj4jtFLyJpCHAw8ChwJHCupDOB+aTaylpSgnmk5LCVbEo6K1rFR5KGYHktIja0sX/r608AJgAMHjy4aLHNzKwD5Woi78uf/ya1h/Qtie1c9AKSdgZ+BlyYx+C6DtiX9ALjajYfar4qImJSRDRHRHNTU1O1L2dm1jDarYlExNcAJJ0SEbeVbpN0SpGTS9qOlEBujog78nlfKtn+IzY10q8C9io5fFCO0U78VaCfpG1zbaR0fzMzq4EibSKXFoxtJrdZTAaeLn30JWmPkt3+FngqL88ATpW0vaShwDDgMWAeMCz3xOpLanyfEREBzAXG5ePHA3cXuB8zM+si5dpEjiMNujhQ0jUlm3Yh9bzqyJGkYeOflLQox74MnCZpBBCkWRI/B5BHB54OLM3nPyciNuaynAvMAvoAUyJiST7fxcA0SZcDj5OSlpmZ1Ui5Lr7/Q2r4PglYUBJ/A/hiRyeOiIfZNIx8qZlljrkCuKKN+My2jouI5aTeW2Zm1g3KtYk8ATwh6eaSHlBmZmbvKfc4a3pEfBJ4XNIWg0JFxEFVLZmZmfV45R5nXZB/nlCLgpiZWf0p9zhrdV7838CDEfFsbYpkZmb1osjYWYOB6/Nb5wuAB4GHImJRuYPMzKz36/A9kYiYGBGjSAMjPgRcxOa9tczMrEF1WBOR9FXSOx87k97F+GdSMjEzswZX5HHWJ0gv//0SeAD4bUSsL3+ImZk1giKPsw4hNa4/RprT40lJD1e7YGZm1vMVeZx1IPDXwEeAZtKw7H6cZWZmhR5nXUnqkXUNMC8i/lTdIpmZWb3oMIlEhF82NDOzNhUZCt7MzKxNTiJmZlaxTiURSdtI2qVahTEzs/rSYRKRdIukXSTtRJqFcKmki6pfNDMz6+mK9M4aHhGvS/o0cA9wCWnYk29VtWRmZtYp+lpb8wAmMXGLGT26RJHHWdtJ2g44mTS3+Z9IU9uamVmDK5JErifNhb4T8KCkvYHXOzpI0l6S5kpaKmmJpAtyvL+k2ZKezT93zXFJukbSMkmLJR1Scq7xef9nJY0viR8q6cl8zDWS2k/DZmbW5YoMe3JNRAyMiOMjeQE4psC5NwD/FBHDgcOBcyQNJz0OmxMRw4A5eR3gOGBY/kwAroOUdICJwEjSfOoTWxJP3ufskuPGFCiXmZl1kSIN6wMkTZZ0T14fDozv4DAiYnVELMzLbwBPAwOBscDUvNtU0mMycvymnKgeAfpJ2gP4GDA7ItZExFpgNjAmb9slIh6JiABuKjmXmZnVQJHHWTcCs4A98/rvgQs7c5E8odXBwKPAgJJZE18EBuTlgaRxuVqszLFy8ZVtxNu6/gRJ8yXNf+WVVzpTdDMzK6NIEtk9IqYD7wJExAZgY9ELSNoZ+BlwYURs1paSaxBVb6SPiEkR0RwRzU1NTdW+nJlZwyiSRN6StBv5l72kw4F1RU6ee3X9DLg5Iu7I4Zfyoyjyz5dzfBWwV8nhg3KsXHxQG3EzM6uRIu+JfAmYAewr6T+BJmBcRwflnlKTgacj4jslm2aQ2lSuzD/vLomfK2kaqRF9XUSsljQL+NeSxvTRwKURsUbS6zmpPQqcCXy/wP1YL1eur3wR1epPb9YbFRnFd6GkjwD7AwKeKTgc/JHAGaRJrBbl2JdJyWO6pLOAF4BP5m0zgeOBZcDbwGfz9ddI+gYwL+/39YhYk5e/QGqz2ZH0IuQ9BcpldWBrE0E9Xruj6zq5WU9UZFKqc0iPo5bk9V0lnRYRPyh3XEQ8TEo6bTm2jf0DOKedc00BprQRnw8cWP4OrLt0ZyLoLt3xxrBZdyryOOvsiLi2ZSUi1ko6GyibRMxsc42YVK33K5JE+khSrikgqQ/Qt7rFsnrgX4q15e/beqIiSeRe4FZJ1+f1z+WYmZk1uCJJ5GJS4vh8Xp8N3FC1EllN+a9bM9saRXpnvSvpRuC+iHim+kUys65Wrz2/3FGh5yvSO+sk0twhfYGhkkaQutmeVO3CWddwbcPMqqXI46yJpNFz7weIiEWShlazUGZWW1vzh4ZrBI2tyLAnf4qI1sOc+L8aMzMrVBNZIul0UlffYcD5wG+qWywzqxd+XNrYitREzgM+CKwHbiENvnhBNQtlZmb1oUhN5OMR8RXgKy0BSacAt1WtVLYZ/6VnZj1VkZrIpQVjZmbWYNqtiUg6jjSq7kBJ15Rs2oU0f7qZmTW4co+z/geYD5wELCiJvwF8sZqFMjOz+tBuEomIJ4AnJN1ScP4Q2wpu9zCzelSkYf0wSZcBe+f9RZr+Y59qFszMzHq+IklkMunx1QJgY3WLY2Zm9aRIElkXEZ521szMtlAkicyV9C3gDtILh0Cae73cQZKmACcAL0fEgTl2GXA28Ere7csRMTNvuxQ4i1TbOT8iZuX4GOBqoA9wQ0RcmeNDgWnAbqRa0hkR8U6B+zGzXqBeRybubYokkZH5Z3NJLIBRHRx3I/DvwE2t4t+NiH8rDUgaDpxKejN+T+DXkj6QN18LfBRYCcyTNCMilgJX5XNNk/RDUgK6rsD9dAs3nJtZb1RkPpFjKjlxRDwoaUjB3ccC0yJiPfCcpGWkkYMBlkXEcgBJ04Cxkp4mJbHT8z5TgcvowUnEzGrLc5HURpH5RAYA/wrsGRHH5VrDERExucJrnivpTNI7KP8UEWuBgcAjJfuszDGAFa3iI0mPsF6LiA1t7N/WPUwAJgAMHjy4wmKbWW/hR2Fdp8iwJzcCs0iPmQB+D1xY4fWuA/YFRgCrgW9XeJ5OiYhJEdEcEc1NTU21uKSZWUMokkR2j4jpwLsA+a//irr6RsRLEbExIt4FfsSmR1argL1Kdh2UY+3FXwX6Sdq2VdzMzGqoSMP6W5J2I09EJelw0nDwnSZpj4hYnVf/FngqL88AbpH0HVKNZxjwGOnFxmG5J9YqUuP76RERkuYC40g9tMYDd1dSpq7kxnOz3sH/LxdXJIl8ifRLfl9J/wk0kX55lyXpp8DRwO6SVpKm2T06z9EewPPA5wAiYomk6cBS0uCO50TExnyec0mP0/oAUyJiSb7ExcA0SZcDj5NeijQzsxoq0jtroaSPAPuTagbPFBlLKyJOayPc7i/6iLgCuKKN+ExgZhvx5Wx6HGZmZt2gwzaRPAHVjrkGcDJwq6RDql4yMzPr8Yo0rP/fiHhD0lHAsaTahN/HMDOzQkmkpSfWx4EfRcQvgb7VK5KZmdWLIklklaTrgU8BMyVtX/A4MzPr5Yokg0+Sekd9LCJeA/oDF1W1VGZmVheK9M56mzSCb8v6atLb5mZm1kqjDanix1JmZlaxdpNIbvswMzNrV7mayG8BJP2kRmUxM7M6U65NpK+k04EPS/pE640RcUcbx5iZWQMpl0T+Efg00A84sdW2oKSx3czMiultk2W1m0Qi4mHgYUnzt2ICKjMzK6gee3YVGcX3J5LOB/4mrz8A/LDIIIxmZta7FUkiPwC2yz8BziCNnfV/qlUoMzPbUk+c56RIEvmriPhQyfp9kp6oVoHMzKx+FBqAUdK+LSuS9qHC6XHNzKx3KVITuQiYK2k5aVKqvYHPVrVUZmZWFzqsiUTEHNKc5+cD5wH7R8Tcjo6TNEXSy5KeKon1lzRb0rP55645LknXSFomaXHppFeSxuf9n5U0viR+qKQn8zHXSOp5DwvNzHq5QmNnRcT6iFicP+sLnvtGYEyr2CXAnIgYBszJ6wDHkRLVMGACedIrSf1Jc7OPJE2FO7El8eR9zi45rvW1zMysyqo2AGNEPAisaRUeC0zNy1NJ0+22xG+K5BGgn6Q9gI8BsyNiTUSsBWYDY/K2XSLikYgI4KaSc5mZWY3UehTfAXkoeYAXgQF5eSCwomS/lTlWLr6yjXibJE2QNF/S/FdeeWXr7sDMzN7TYRKRNKdIrLNyDaImr19GxKSIaI6I5qamplpc0sysIZQbCn6H3Caxu6Rdc6N4f0lDKPNXfwdeyo+iyD9fzvFVwF4l+w3KsXLxQW3EzcyshsrVRD4HLAAOyD9bPncD/17h9WYALT2sxudztcTPzL20DgfW5cdes4DROYntCowGZuVtr0s6PPfKOrPkXGZmViPlBmC8Grha0nkR8f3OnljST4GjSTWZlaReVlcC0yWdBbxAmr8dYCZwPLAMeJv8HkpErJH0DWBe3u/rEdHSWP8FUg+wHYF78sfMzGqoyBzr35f0YWBI6f4RcVMHx53WzqZj29g3gHPaOc8UYEob8fnAgeXKYGZm1dVhEskzG+4LLGLTcCct3WrNzKyBFRn2pBkYnmsLZmZm7ynynshTwF9UuyBmZlZ/itREdgeWSnoMeG/Ik4g4qWqlMjOzulAkiVxW7UKYmVl9KtI764FaFMTMzOpPkd5Zb7BpeJK+pKly34qIXapZMDMz6/mK1ETe17Kc3w4fCxxezUKZmVl96NQovnmo9rtIQ7SbmVmDK/I46xMlq9uQ3hv5Y9VKZGZmdaNI76wTS5Y3AM+THmmZmVmDK9Im8tlaFMTMzOpPkUmpBkm6U9LL+fMzSYM6Os7MzHq/Ig3rPybN97Fn/vw8x8zMrMEVSSJNEfHjiNiQPzcCnmPWzMwKJZFXJX1GUp/8+QzwarULZmZmPV+RJPIPpBkIXwRWA+PIMw+amVljK9I76wXAI/aamdkWivTOGirpO5LukDSj5bM1F5X0vKQnJS2SND/H+kuaLenZ/HPXHJekayQtk7RY0iEl5xmf939W0vitKZOZmXVekZcN7wImk3plvduF1z4mIv5Qsn4JMCcirpR0SV6/GDgOGJY/I4HrgJGS+gMTSW/QB7BA0oyIWNuFZTQzszKKJJE/RsQ1VS9Jegv+6Lw8FbiflETGAjfl6XkfkdRP0h5539kRsQZA0mxgDPDTGpTVzMwolkSuljQR+BWbz2y4cCuuG8CvJAVwfURMAgZExOq8/UVgQF4eCKwoOXZljrUX34KkCcAEgMGDB29Fsc3MrFSRJPKXwBnAKDY9zoq8XqmjImKVpD8HZkv6XenGiIicYLpETlKTAJqbm7vsvGZmja5IEjkF2Cci3umqi0bEqvzzZUl3AocBL0naIyJW58dVL+fdVwF7lRw+KMdWsenxV0v8/q4qo5mZdazIeyJPAf266oKSdpL0vpZlYHS+xgygpYfVeODuvDwDODP30jocWJcfe80CRkvaNffkGp1jZmZWI0VqIv2A30max+ZtIpW+OzIAuDNNksi2wC0RcW8+/3RJZwEvkF5wBJgJHA8sA94mv+gYEWskfQOYl/f7eksju5mZ1UaRJDKxKy8YEcuBD7URfxU4to14AOe0c64pwJSuLJ+ZmRVX5I31B0rXJR0FnAY80PYRZmbWKIrURJB0MHA6qZH9OeBn1SyUmZnVh3aTiKQPkGocpwF/AG4FFBHH1KhsZmbWw5WrifwOeAg4ISKWAUj6Yk1KZWZmdaFcF99PkIZ+nyvpR5KOBVSbYpmZWT1oN4lExF0RcSpwADAXuBD4c0nXSRpdqwKamVnP1eHLhhHxVkTcEhEnkt4Kf5w0MKKZmTW4Im+svyci1kbEpIjY4n0OMzNrPJ1KImZmZqWcRMzMrGJOImZmVjEnETMzq5iTiJmZVcxJxMzMKuYkYmZmFXMSMTOzijmJmJlZxZxEzMysYnWfRCSNkfSMpGWSLunu8piZNZK6TiKS+gDXAscBw4HTJA3v3lKZmTWOuk4iwGHAsohYHhHvANOAsd1cJjOzhlFojvUebCCwomR9JTCy9U6SJgAT8uqbkp7p4nLsTppCuJH5O/B3AP4Oeuz967KtnlNw77aC9Z5EComIScCkap1f0vyIaK7W+euBvwN/B+DvoBHvv94fZ60C9ipZH5RjZmZWA/WeROYBwyQNldQXOBWY0c1lMjNrGHX9OCsiNkg6F5gF9AGmRMSSbihK1R6V1RF/B/4OwN9Bw92/IqK7y2BmZnWq3h9nmZlZN3ISMTOzijmJdJKkHSQ9JukJSUskfS3Hh0p6NA+/cmtu6O+1JPWR9LikX+T1Rrv/5yU9KWmRpPk51l/SbEnP5p+7dnc5q0lSP0m3S/qdpKclHdFI34Gk/fO/f8vndUkXNtJ3AE4ilVgPjIqIDwEjgDGSDgeuAr4bEfsBa4GzurGMtXAB8HTJeqPdP8AxETGi5L2AS4A5ETEMmJPXe7OrgXsj4gDgQ6T/HhrmO4iIZ/K//wjgUOBt4E4a6DsAJ5FOi+TNvLpd/gQwCrg9x6cCJ3dD8WpC0iDg48ANeV000P2XMZZ079DLvwNJ7wf+BpgMEBHvRMRrNNB30MqxwH9FxAs02HfgJFKB/ChnEfAyMBv4L+C1iNiQd1lJGpKlt/oe8C/Au3l9Nxrr/iH94fArSQvysDoAAyJidV5+ERjQPUWriaHAK8CP82PNGyTtRGN9B6VOBX6alxvqO3ASqUBEbMxV2EGkQSAP6OYi1YykE4CXI2JBd5elmx0VEYeQRpA+R9LflG6M1He+N/ef3xY4BLguIg4G3qLVY5sG+A4AyO1/JwG3td7WCN+Bk8hWyNX3ucARQD9JLS9v9ubhV44ETpL0PGnU5FGkZ+ONcv8ARMSq/PNl0nPww4CXJO0BkH++3H0lrLqVwMqIeDSv305KKo30HbQ4DlgYES/l9Yb6DpxEOklSk6R+eXlH4KOkBsW5wLi823jg7u4pYXVFxKURMSgihpCq8PdFxKdpkPsHkLSTpPe1LAOjgadIQ+6Mz7v16u8gIl4EVkjaP4eOBZbSQN9BidPY9CgLGuw78BvrnSTpIFJjWR9SEp4eEV+XtA/pL/P+wOPAZyJiffeVtPokHQ38c0Sc0Ej3n+/1zry6LXBLRFwhaTdgOjAYeAH4ZESs6aZiVp2kEaTOFX2B5cBnyf9P0DjfwU7AfwP7RMS6HGus/w6cRMzMrFJ+nGVmZhVzEjEzs4o5iZiZWcWcRMzMrGJOImZmVjEnEasKSW92vNdWnf9CSX/WFdeTtL2kX+eRWD/VNSXc4hpfrsZ5u4Kkox0MB/AAAAU5SURBVFtGY67R9YZIOr1W17PqchKxenUh8Gcd7lXMwQB5RNZbu+icrfXYJLK1SkYqKGoI0KkkUsE1rEacRKxmJO0r6d48aOFDkg7I8RslXSPpN5KWSxqX49tI+kGer2K2pJmSxkk6H9gTmCtpbsn5r8jzvDwiaYtB7/I8D3dJWpz3OUjSnwP/AfxVrons2+qY8yUtzcdMy7GdJE1RmlfmcUljc/zvJd2R7/FZSd/M8SuBHfP5b86xz+TjF0m6XlKfHH+zrfuQNEDSnTn+hKQPlztPmX+D/XKt6wlJC0vud2dtmhvk5jwyM5L+n6R5kp6SNKkkfr+k7ynNpXKBpEMlPZD/bWeVDPvR1vWuBP46l/mLSgOafitfZ7Gkz+Vjj87/ncwAlubv/Zf5XE9Vq9ZonRQR/vjT5R/gzTZic4BheXkkacgUgBtJg9dtAwwHluX4OGBmjv8FaZ6ScXnb88DuJecO4MS8/E3gq21c//vAxLw8CliUl48GftHOffwPsH1e7pd//ivpjXyAfsDvgZ2Avye9uf1+YAfS28p7tf4+gP8F/BzYLq//ADiz3H0AtwIX5uU++RrtnqfMv8ujwN/m5R1ItbmjgXWkMc+2AX5LGmASoH/JsT8pKdv9wA/y8nbAb4CmvP4pYEoH1/tFyXknlNzn9sB80ijBR5MGdhyat/0d8KOS497f3f+d+xO4img1IWln4MPAbfmPWUi/MFrcFRHvkv7ibKlFHAXcluMvltY62vAO0PJcfwFpTLPWjiL9IiIi7pO0m6RdOij6YuBmSXcBd+XYaNIglP+c13cgDXEBaTKiluEvlgJ7AytanfNY0iRG8/J3sSObBulr7z5GAWfmsm8E1kk6o8x5tqA03tfAiLgzn+ePOQ7wWESszOuLSI+cHgaOkfQvpF/+/YElpMQFKbEB7A8cCMzO5+oDrO7geqVGAwe11EBJCXJY/i4ei4jncvxJ4NuSriIloYfau1erHScRq5VtSHOOjGhne+k4W1v8lingT5H/PAU20nX/bX+cNPnSicBXJP1lLt/fRcQzpTtKGsnm99FeOQRMjYhL29jWmfsod57O2qLcknYg1W6aI2KFpMtICbPFWyXlWBIRR2xWuDxIZQECzouIWa2OP7rkGkTE7yUdAhwPXC5pTkR8veA1rErcJmI1ERGvA89JOgXSbIiSPtTBYf8J/J1S28gA0uONFm8ARX9JtXgI+HS+/tHAH3K52iRpG9LjqLnAxaS/kHcGZgHnlbQPHFzg2n+StF1engOMU2qPaWmr2buD4+cAn8/791GaWbDd80i6SdJhpSeIiDeAlZJOzvtsr5Iebm1oSRh/yDXJce3s9wzQJOmIfN7tJH2wzPVa/9vNAj7f8v1I+oDSwIabkbQn8HZE/AfwLdLQ89bNnESsWv5M0sqSz5dIv8DPkvQE6bHI2A7O8TPSvBVLSY3fC0nP7gEmAfd28IirtcuAQyUtJjXuji+/O32A/5D0JGlk4msizSHzDVI7wGJJS/J6Rybl/W+OiKXAV0kzIy4mzY65RwfHX0B6tPQk6THX8A7OcxCpPae1M4Dz8/6/IbU1tSnf649Iw9zPAua1s987pARzVf63XUR6dNne9RYDG3MD+RdJIwEvBRZKegq4nrZrYH8JPJYft00ELm+v7FY7HsXXejRJO0fEm0rDaz8GHBlpLgtrR27nmRwRp3R3Waz3cxKxHk3S/aQeUH2Bb0bEjd1aIDPbjJOImZlVzG0iZmZWMScRMzOrmJOImZlVzEnEzMwq5iRiZmYV+/+untthCgVo9wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Part where you can save/load filtered dataset"
      ],
      "metadata": {
        "id": "rUHd1nwQ7byR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file_path = 'filtered_transl.csv'"
      ],
      "metadata": {
        "id": "Dw294POuHaaX"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Write filtered data\n",
        "with open(file_path, \"w\", newline='', encoding=\"utf-8\") as out:\n",
        "        out.write(pd.DataFrame(data=filtered_small).to_csv(index=False))\n"
      ],
      "metadata": {
        "id": "KXL0kTWq7baW"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Choose load or process filtered data \n",
        "load_data = True #@param {type:\"boolean\"}\n",
        "if not load_data:\n",
        "    lines_eng, lines_rus = filtered_lines_filter_q['eng'], filtered_lines_filter_q['rus']\n",
        "else:\n",
        "    # Load filtered data\n",
        "    df = pd.read_csv(file_path, index_col=False)\n",
        "    lines_eng, lines_rus = list(df['eng']), list(df['rus'])"
      ],
      "metadata": {
        "id": "T5V2wnZqj5Pd"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "y8AKm26RlbjV",
        "outputId": "bb5a9d0f-eedb-44a2-bb3b-e58adb893aa2"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>rus</th>\n",
              "      <th>eng</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>O НАС Новости Референции Контакт\\n</td>\n",
              "      <td>About us News References Contacts\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Ваш комфорт - наша обязанность…\\n</td>\n",
              "      <td>Your comfort, our obligation…\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Достаточно просто попросить нас об этом.\\n</td>\n",
              "      <td>All you need to do is just ask for it.\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Этот автор еще не заполнил сведений о себе.\\n</td>\n",
              "      <td>This author has not yet filled in any details.\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>К сожалению, эта страница не найдена!\\n</td>\n",
              "      <td>Oops, This Page Could Not Be Found!\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>799467</th>\n",
              "      <td>Приглашения на мероприятия дипломатических мис...</td>\n",
              "      <td>Invitations to Embassy’s an Consulate’s events...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>799468</th>\n",
              "      <td>Доступ к другим группам влияния + +\\n</td>\n",
              "      <td>Access to members of other communities + +\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>799469</th>\n",
              "      <td>Специальная цена на рекламу на сайте РСП + -\\n</td>\n",
              "      <td>Special rate for ad on RBC website + -\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>799470</th>\n",
              "      <td>Специальная цена на рекламу в журнале «Деловые...</td>\n",
              "      <td>Special rate for ad in RBC magazine + -\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>799471</th>\n",
              "      <td>Специальные тарифы на спонсорство + -\\n</td>\n",
              "      <td>Preferential rate for sponsorship + -\\n</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>799472 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                      rus                                                eng\n",
              "0                      O НАС Новости Референции Контакт\\n                About us News References Contacts\\n\n",
              "1                       Ваш комфорт - наша обязанность…\\n                    Your comfort, our obligation…\\n\n",
              "2              Достаточно просто попросить нас об этом.\\n           All you need to do is just ask for it.\\n\n",
              "3           Этот автор еще не заполнил сведений о себе.\\n   This author has not yet filled in any details.\\n\n",
              "4                 К сожалению, эта страница не найдена!\\n              Oops, This Page Could Not Be Found!\\n\n",
              "...                                                   ...                                                ...\n",
              "799467  Приглашения на мероприятия дипломатических мис...  Invitations to Embassy’s an Consulate’s events...\n",
              "799468              Доступ к другим группам влияния + +\\n       Access to members of other communities + +\\n\n",
              "799469     Специальная цена на рекламу на сайте РСП + -\\n           Special rate for ad on RBC website + -\\n\n",
              "799470  Специальная цена на рекламу в журнале «Деловые...          Special rate for ad in RBC magazine + -\\n\n",
              "799471            Специальные тарифы на спонсорство + -\\n            Preferential rate for sponsorship + -\\n\n",
              "\n",
              "[799472 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Just to check everything is loaded ok\n",
        "lines_eng[:5], lines_rus[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GJfH5FPpIwMc",
        "outputId": "955a1a66-8d0d-4261-b079-09631c21e464"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['About us News References Contacts\\n',\n",
              "  'Your comfort, our obligation…\\n',\n",
              "  'All you need to do is just ask for it.\\n',\n",
              "  'This author has not yet filled in any details.\\n',\n",
              "  'Oops, This Page Could Not Be Found!\\n'],\n",
              " ['O НАС Новости Референции Контакт\\n',\n",
              "  'Ваш комфорт - наша обязанность…\\n',\n",
              "  'Достаточно просто попросить нас об этом.\\n',\n",
              "  'Этот автор еще не заполнил сведений о себе.\\n',\n",
              "  'К сожалению, эта страница не найдена!\\n'])"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Take only N elements from dataset\n",
        "N = 100000\n",
        "lines_eng, lines_rus = lines_eng[:N], lines_rus[:N]"
      ],
      "metadata": {
        "id": "gq6ZP3zN3k31"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Models"
      ],
      "metadata": {
        "id": "jG4-OdSuH6CN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "MZC0vd52faCk"
      },
      "outputs": [],
      "source": [
        "from torchtext.data.utils import get_tokenizer\n",
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from typing import Iterable, List\n",
        "\n",
        "\n",
        "SRC_LANGUAGE = 'eng'\n",
        "TGT_LANGUAGE = 'rus'\n",
        "\n",
        "# Place-holders\n",
        "token_transform = {}\n",
        "vocab_transform = {}\n",
        "\n",
        "\n",
        "# Create source and target language tokenizer. Make sure to install the dependencies.\n",
        "token_transform[SRC_LANGUAGE] = get_tokenizer('spacy', language='ru_core_news_sm')\n",
        "token_transform[TGT_LANGUAGE] = get_tokenizer('spacy', language='en_core_web_sm')\n",
        "\n",
        "\n",
        "# helper function to yield list of tokens\n",
        "def yield_tokens(data_iter: Iterable, language: str) -> List[str]:\n",
        "    for data_sample in data_iter:\n",
        "        yield token_transform[language](data_sample)\n",
        "\n",
        "# Define special symbols and indices\n",
        "UNK_IDX, PAD_IDX, BOS_IDX, EOS_IDX = 0, 1, 2, 3\n",
        "# Make sure the tokens are in order of their indices to properly insert them in vocab\n",
        "special_symbols = ['<unk>', '<pad>', '<bos>', '<eos>']\n",
        " \n",
        "for train_iter, ln in [(lines_eng, 'eng'), (lines_rus, 'rus')]:\n",
        "    # Create torchtext's Vocab object \n",
        "    vocab_transform[ln] = build_vocab_from_iterator(yield_tokens(train_iter, ln),\n",
        "                                                    min_freq=1,\n",
        "                                                    specials=special_symbols,\n",
        "                                                    special_first=True)\n",
        "\n",
        "# Set UNK_IDX as the default index. This index is returned when the token is not found. \n",
        "# If not set, it throws RuntimeError when the queried token is not found in the Vocabulary. \n",
        "for ln in [SRC_LANGUAGE, TGT_LANGUAGE]:\n",
        "  vocab_transform[ln].set_default_index(UNK_IDX)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Seq2Seq Network using LSTM"
      ],
      "metadata": {
        "id": "JDj48w3bcZ1P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "N_train = int(N * 0.9)\n",
        "\n",
        "raw_pairs = list(zip(lines_eng, lines_rus))\n",
        "train_iter = raw_pairs[:N_train]\n",
        "test_iter = raw_pairs[N_train:] \n",
        "N_train"
      ],
      "metadata": {
        "id": "Dw6gYmtjy3sa",
        "outputId": "ac2edfc1-ccf3-46cf-8634-73f32e8f71c1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "90000"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from __future__ import unicode_literals, print_function, division\n",
        "from io import open\n",
        "import unicodedata\n",
        "import string\n",
        "import re\n",
        "import random\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch import optim\n",
        "import torch.nn.functional as F\n",
        "SOS_token = 0\n",
        "EOS_token = 1\n",
        "\n",
        "\n",
        "class Lang:\n",
        "    def __init__(self, name):\n",
        "        self.name = name\n",
        "        self.word2index = {}\n",
        "        self.word2count = {}\n",
        "        self.index2word = {0: \"SOS\", 1: \"EOS\"}\n",
        "        self.n_words = 2  # Count SOS and EOS\n",
        "\n",
        "    def addSentence(self, sentence):\n",
        "        for word in sentence.split(' '):\n",
        "            self.addWord(word)\n",
        "\n",
        "    def addWord(self, word):\n",
        "        if word not in self.word2index:\n",
        "            self.word2index[word] = self.n_words\n",
        "            self.word2count[word] = 1\n",
        "            self.index2word[self.n_words] = word\n",
        "            self.n_words += 1\n",
        "        else:\n",
        "            self.word2count[word] += 1\n",
        "\n",
        "\n",
        "def unicodeToAscii(s):\n",
        "    return ''.join(\n",
        "        c for c in unicodedata.normalize('NFD', s)\n",
        "        if unicodedata.category(c) != 'Mn'\n",
        "    )\n",
        "\n",
        "\n",
        "# Lowercase, trim, and remove non-letter characters\n",
        "def normalizeString(s):\n",
        "    s = unicodeToAscii(s.lower().strip())\n",
        "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
        "    s = re.sub(r\"[^a-zA-ZА-Яа-я.!?]+\", r\" \", s)\n",
        "    return s\n",
        "\n",
        "\n",
        "def readLangs(lang1, lang2, pre_pairs):\n",
        "    print(\"Reading pairs...\")\n",
        "    # normalize every pair\n",
        "    pairs = [[normalizeString(s) for s in s] for s in pre_pairs]\n",
        "    input_lang = Lang(lang1)\n",
        "    output_lang = Lang(lang2)\n",
        "    return input_lang, output_lang, pairs\n",
        "\n",
        "def prepareData(lang1, lang2, pre_pairs):\n",
        "    input_lang, output_lang, pairs = readLangs(lang1, lang2, pre_pairs)\n",
        "    print(f\"Read {len(pairs)} sentence pairs\")\n",
        "    print(f\"Trimmed to {len(pairs)} sentence pairs\")\n",
        "    print(\"Adding words...\")\n",
        "    max_length = 0\n",
        "    for pair in pairs:\n",
        "        max_length = max(len(pair[0]), len(pair[1]), max_length)\n",
        "        input_lang.addSentence(pair[0])\n",
        "        output_lang.addSentence(pair[1])\n",
        "    print(\"Counted words:\")\n",
        "    print(input_lang.name, input_lang.n_words)\n",
        "    print(output_lang.name, output_lang.n_words)\n",
        "    return input_lang, output_lang, pairs, max_length\n",
        "\n",
        "input_lang, output_lang, pairs, max_length = prepareData('eng', 'rus', raw_pairs)\n",
        "print(random.choice(pairs))\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "id": "9l7usMOfeaNE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ee047d8-a43f-4b20-f7ff-729a9b319af6"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading pairs...\n",
            "Read 100000 sentence pairs\n",
            "Trimmed to 100000 sentence pairs\n",
            "Adding words...\n",
            "Counted words:\n",
            "eng 29751\n",
            "rus 72121\n",
            "['the total number of neutrons and protons is called the mass number .', 'общее число протонов и неитронов называется массовым числом .']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def indexesFromSentence(lang, sentence):\n",
        "    return [lang.word2index[word] for word in sentence.split(' ')]\n",
        "\n",
        "\n",
        "def tensorFromSentence(lang, sentence):\n",
        "    indexes = indexesFromSentence(lang, sentence)\n",
        "    indexes.append(EOS_token)\n",
        "    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)\n",
        "\n",
        "\n",
        "def tensorsFromPair(pair):\n",
        "    input_tensor = tensorFromSentence(input_lang, pair[0])\n",
        "    target_tensor = tensorFromSentence(output_lang, pair[1])\n",
        "    return (input_tensor, target_tensor)"
      ],
      "metadata": {
        "id": "QgXjWDBpicxm"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class EncoderRNN(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size):\n",
        "        super(EncoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
        "\n",
        "    def forward(self, input, hidden):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        output = embedded\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "        return output, hidden\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ],
      "metadata": {
        "id": "_TEEuFdhciRd"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class AttnDecoderRNN(nn.Module):\n",
        "    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=max_length):\n",
        "        super(AttnDecoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.output_size = output_size\n",
        "        self.dropout_p = dropout_p\n",
        "        self.max_length = max_length\n",
        "\n",
        "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
        "        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n",
        "        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
        "        self.dropout = nn.Dropout(self.dropout_p)\n",
        "        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n",
        "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
        "\n",
        "    def forward(self, input, hidden, encoder_outputs):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        embedded = self.dropout(embedded)\n",
        "\n",
        "        attn_weights = F.softmax(\n",
        "            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)\n",
        "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
        "                                 encoder_outputs.unsqueeze(0))\n",
        "\n",
        "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
        "        output = self.attn_combine(output).unsqueeze(0)\n",
        "\n",
        "        output = F.relu(output)\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "\n",
        "        output = F.log_softmax(self.out(output[0]), dim=1)\n",
        "        return output, hidden, attn_weights\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ],
      "metadata": {
        "id": "gplYekfRcuny"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "teacher_forcing_ratio = 0.5\n",
        "def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=max_length):\n",
        "    encoder_hidden = encoder.initHidden()\n",
        "\n",
        "    encoder_optimizer.zero_grad()\n",
        "    decoder_optimizer.zero_grad()\n",
        "\n",
        "    input_length = input_tensor.size(0)\n",
        "    target_length = target_tensor.size(0)\n",
        "\n",
        "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
        "\n",
        "    loss = 0\n",
        "\n",
        "    for ei in range(input_length):\n",
        "        encoder_output, encoder_hidden = encoder(\n",
        "            input_tensor[ei], encoder_hidden)\n",
        "        encoder_outputs[ei] = encoder_output[0, 0]\n",
        "\n",
        "    decoder_input = torch.tensor([[SOS_token]], device=device)\n",
        "\n",
        "    decoder_hidden = encoder_hidden\n",
        "\n",
        "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
        "\n",
        "    if use_teacher_forcing:\n",
        "        # Teacher forcing: Feed the target as the next input\n",
        "        for di in range(target_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            loss += criterion(decoder_output, target_tensor[di])\n",
        "            decoder_input = target_tensor[di]  # Teacher forcing\n",
        "\n",
        "    else:\n",
        "        # Without teacher forcing: use its own predictions as the next input\n",
        "        for di in range(target_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            topv, topi = decoder_output.topk(1)\n",
        "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
        "\n",
        "            loss += criterion(decoder_output, target_tensor[di])\n",
        "            if decoder_input.item() == EOS_token:\n",
        "                break\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    encoder_optimizer.step()\n",
        "    decoder_optimizer.step()\n",
        "\n",
        "    return loss.item() / target_length"
      ],
      "metadata": {
        "id": "Wg8k8Y3adIi1"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "import math\n",
        "\n",
        "\n",
        "def asMinutes(s):\n",
        "    m = math.floor(s / 60)\n",
        "    s -= m * 60\n",
        "    return '%dm %ds' % (m, s)\n",
        "\n",
        "\n",
        "def timeSince(since, percent):\n",
        "    now = time.time()\n",
        "    s = now - since\n",
        "    es = s / (percent)\n",
        "    rs = es - s\n",
        "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))"
      ],
      "metadata": {
        "id": "RbZ5vb3TdOlA"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def Trainer(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):\n",
        "    start = time.time()\n",
        "    plot_losses = []\n",
        "    print_loss_total = 0  # Reset every print_every\n",
        "    plot_loss_total = 0  # Reset every plot_every\n",
        "\n",
        "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
        "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
        "    training_pairs = [tensorsFromPair(random.choice(pairs))\n",
        "                      for i in range(n_iters)]\n",
        "    criterion = nn.NLLLoss()\n",
        "\n",
        "    for iter in range(1, n_iters + 1):\n",
        "        training_pair = training_pairs[iter - 1]\n",
        "        input_tensor = training_pair[0]\n",
        "        target_tensor = training_pair[1]\n",
        "\n",
        "        loss = train(input_tensor, target_tensor, encoder,\n",
        "                     decoder, encoder_optimizer, decoder_optimizer, criterion)\n",
        "        print_loss_total += loss\n",
        "        plot_loss_total += loss\n",
        "\n",
        "        if iter % print_every == 0:\n",
        "            print_loss_avg = print_loss_total / print_every\n",
        "            print_loss_total = 0\n",
        "            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),\n",
        "                                         iter, iter / n_iters * 100, print_loss_avg))\n",
        "\n",
        "        if iter % plot_every == 0:\n",
        "            plot_loss_avg = plot_loss_total / plot_every\n",
        "            plot_losses.append(plot_loss_avg)\n",
        "            plot_loss_total = 0\n",
        "\n",
        "    showPlot(plot_losses)"
      ],
      "metadata": {
        "id": "zZIZEl_VdTY4"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.switch_backend('agg')\n",
        "import matplotlib.ticker as ticker\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "def showPlot(points):\n",
        "    plt.figure()\n",
        "    fig, ax = plt.subplots()\n",
        "    # this locator puts ticks at regular intervals\n",
        "    loc = ticker.MultipleLocator(base=0.2)\n",
        "    ax.yaxis.set_major_locator(loc)\n",
        "    plt.plot(points)"
      ],
      "metadata": {
        "id": "kIOzlia4dUFy"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(encoder, decoder, sentence, max_length=max_length):\n",
        "    with torch.no_grad():\n",
        "        input_tensor = tensorFromSentence(input_lang, sentence)\n",
        "        input_length = input_tensor.size()[0]\n",
        "        encoder_hidden = encoder.initHidden()\n",
        "\n",
        "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
        "\n",
        "        for ei in range(input_length):\n",
        "            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n",
        "                                                     encoder_hidden)\n",
        "            encoder_outputs[ei] += encoder_output[0, 0]\n",
        "\n",
        "        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS\n",
        "\n",
        "        decoder_hidden = encoder_hidden\n",
        "\n",
        "        decoded_words = []\n",
        "        decoder_attentions = torch.zeros(max_length, max_length)\n",
        "\n",
        "        for di in range(max_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            decoder_attentions[di] = decoder_attention.data\n",
        "            topv, topi = decoder_output.data.topk(1)\n",
        "            if topi.item() == EOS_token:\n",
        "                decoded_words.append('<EOS>')\n",
        "                break\n",
        "            else:\n",
        "                decoded_words.append(output_lang.index2word[topi.item()])\n",
        "\n",
        "            decoder_input = topi.squeeze().detach()\n",
        "\n",
        "        return decoded_words, decoder_attentions[:di + 1]\n",
        "\n",
        "def evaluateRandomly(encoder, decoder, n=10):\n",
        "    for i in range(n):\n",
        "        pair = random.choice(pairs)\n",
        "        print('>', pair[0])\n",
        "        print('=', pair[1])\n",
        "        output_words, attentions = evaluate(encoder, decoder, pair[0])\n",
        "        output_sentence = ' '.join(output_words)\n",
        "        print('<', output_sentence)\n",
        "        print('')"
      ],
      "metadata": {
        "id": "0IKdojV5dXAz"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hidden_size = 256\n",
        "encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)\n",
        "attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1).to(device)\n",
        "\n",
        "Trainer(encoder1, attn_decoder1, 75000, print_every=5000)"
      ],
      "metadata": {
        "id": "VIPicm1cdkGx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "outputId": "54f00d15-25e6-435a-ef32-4623fba2f2bb"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6m 49s (- 95m 36s) (5000 6%) 6.7488\n",
            "13m 36s (- 88m 29s) (10000 13%) 6.5405\n",
            "20m 30s (- 82m 3s) (15000 20%) 6.3895\n",
            "27m 24s (- 75m 23s) (20000 26%) 6.2388\n",
            "34m 20s (- 68m 41s) (25000 33%) 6.1788\n",
            "41m 15s (- 61m 53s) (30000 40%) 6.0816\n",
            "48m 19s (- 55m 13s) (35000 46%) 5.9749\n",
            "55m 19s (- 48m 24s) (40000 53%) 5.8661\n",
            "62m 17s (- 41m 31s) (45000 60%) 5.8182\n",
            "69m 14s (- 34m 37s) (50000 66%) 5.7662\n",
            "76m 10s (- 27m 42s) (55000 73%) 5.6841\n",
            "83m 9s (- 20m 47s) (60000 80%) 5.5899\n",
            "90m 6s (- 13m 51s) (65000 86%) 5.5663\n",
            "97m 4s (- 6m 56s) (70000 93%) 5.5249\n",
            "104m 1s (- 0m 0s) (75000 100%) 5.4582\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deXgV5dn/v/dZspCQhH2HgCD7jiKCIooL4tpal9a1Ul+X2vqzr5Vata1Li7V1e+tSl9rF1tqquIsLouICCAIimyCEfQsECAlJzvL8/ph55szyzDlzkjnJOeH+XFcuzpmZM/MkId+55/vcz32TEAIMwzBM7hNo6QEwDMMw/sCCzjAM00pgQWcYhmklsKAzDMO0EljQGYZhWgmhlrpwx44dRXl5eUtdnmEYJidZsmRJpRCik2pfiwl6eXk5Fi9e3FKXZxiGyUmIaJPbPrZcGIZhWgks6AzDMK0EFnSGYZhWQkpBJ6KBRLTM9HWQiG5yOfYYIooS0QX+D5VhGIZJRspJUSHEWgCjAICIggC2AZhtP07fdx+Ad30eI8MwDOOBdC2XUwB8K4RQzbLeCOAlALubPCqGYRgmbdIV9IsBPG/fSEQ9AJwP4PFkHyaia4hoMREt3rNnT5qXZhiGYZLhWdCJKA/AOQD+q9j9EIBbhRDxZOcQQjwphBgnhBjXqZMyL94zq7YfxJebq5p0DoZhmNZEOguLpgH4UgixS7FvHIB/ExEAdARwJhFFhRCv+DBGJWc+Mh8AUDFreqYuwTAMk1OkI+iXQGG3AIAQoq98TUR/BfBGJsWcYRiGceLJciGiIgCnAnjZtO1aIro2UwNjGIZh0sNThC6EqAHQwbbtCZdjr2z6sBiGYZh04ZWiDMMwrQQWdIZhmFYCCzrDMEwrgQWdYRimlcCCzjAM00pgQWcYhmklsKAzDMO0EljQGYZhWgm+NLggoh8Q0VdEtIKIPiOikZkbMsMwDKPCrwYXGwFMFkJUEdE0AE8CGO/zWBmGYZgkpFOcC3BpcCGE+Mz0dgGAnk0dGMMwDJMevjS4sHE1gLcbNxyGYRimsXiO0E0NLn6R5Jgp0AR9ksv+awBcAwC9e/dOa6AMwzBMctKJ0JM1uAARjQDwNIBzhRB7Vcf42bHIdE5fzsMwDJPrpCPorg0uiKg3tFrplwkhvvFjYF6JxlnQGYZhAP8aXNwJrV76Y3pq42LfR+rCD//6RXNdimEYJqvxpcGFEGIGgBn+Ds0b89dVYmNlDfp2LGqJyzMMw2QNOblS9PXl2y3vp/zhQ8xbu7uFRsMwDJMd5KSg3/j8Use2NTuqW2AkDMMw2UNOCrqKOGe7MAxzhNNqBJ3TFxmGOdJpNYLO2YsMwxzptCJBZ0VnGObIJicFPRggxzY/IvRdB+uwZFNV00/EMAzTAuSmoJNT0P3w0Kc+8BG++/hnqQ9kGIbJQnJS0BV6ntRyEUKg8lB9yvNW10WbMiyGYZgWxa+ORUREjxDRer1z0ZjMDRkIKCN09+Nf+GILxt3zPlZtP5jBUTEMw7QsKQVdCLFWCDFKCDEKwFgAtXB2LJoGYID+dQ2Ax/0eqJmBXds6tiXz0OevrwQArN9zKFNDYhiGaXHStVyUHYsAnAvg70JjAYAyIurmywgVFIaDjm1CCMTiArsP1jn2KRyapMQ4B5JhmBzEr45FPQBsMb3fqm+zQETXENFiIlq8Z8+eNC+dIBKLO7bFhcDv56zBsb+di301DcrPeZ04VZ2fYRgm2/Es6KaORf9t7MX8anARiQsc1anIMjkaF8B7q7TeG/trE4K+eW+tsd3z+VnQGYbJQfzqWLQNQC/T+576towQjcXRt2MRXv9xotOdEDC8FXMcPvXBj1AfTU+gIzG2XBiGyT186VgE4DUAl+vZLscBOCCE2NHk0bkQicURCgSQH0oM35y2GDd54A0mMfeaqh7lCJ1hmBzEr45FbwHYAGA9gKcAXO/zOC1EYwKhIKHANDkaF8KY/PQSke+udk6eSiI8KcowTA7iV8ciAeAGf4fmTiQeRzgYQH44cT8y9xZNJejvrtyJa/6xBM9dPR6TBnR0nj9Ni4ZhGCYbyMmVopGoQNgWoZttkvpoTPk5obvrSzZr9VpWbDugPC4aZ0FnGCb3yElBj8bjCAUDKAiZBV2A9LSXhlQRdgpHpSHKlgvDMLlHTgp6JCYQDhDCwUTeYnV9FDX1Wi0WN8vFPimqqgkDcITOMExu4slDzzbiQiAQICMiB2DJNfcq6G5ZL5y2yDBMLpKTEXo8LpQFuiQpLZcU8MIihmFykdwUdKFuciFxnxS14mq5cITOMEwOkqOCLlzFGADqI+oIO+4xvzzCHjrDMDlIzgp6UsvFxTKJ6aa5m6zLU3IeOsMwuYjXlaJlRPQiEa0hotVENMG2v5SIXiei5US0koiuysxwNeJC3YZOcrhBbbl8s6va0uTCfgZ5k4jySlGGYXIQrxH6wwDmCCEGARgJYLVt/w0AVgkhRgI4CcAf9eqMGSEWF0hioeNwRC3oz35agTMfma/cV9sQNeqg86QowzC5SMq0RSIqBXAigCsBQAjRAMBecFwAaEtaHmExgH0AMtKgU9Y0DyRR9NqGKOau3oX3Vycvm2uOw2+f/bXxmtMWGYbJRbxE6H0B7AHwLBEtJaKn9WJdZv4EYDCA7QBWAPipEMIR5vrR4EK6IdIe6VDkfBCorY/h0Xnr8fyiLY59bmzaV2u85mqLDMPkIl4EPQRgDIDHhRCjAdQAmGk75nQAywB0BzAKwJ+IqMR+Ij8aXEhbRAbo82+dgqmDO1uOqW2IeWojZ47xzZ48Wy4Mw+QiXgR9K4CtQoiF+vsXoQm8masAvKz3FF0PYCOAQf4NM0HcZrm0yQuhvS1Kr2mIJp3YVLWiM8+xsuXCMEwuklLQhRA7AWwhooH6plMArLIdtlnfDiLqAmAgtProvmMIukmBQ0HrtzF/XSVWmrJZvBBwidCv+ftinPfop40ZKsMwTLPitZbLjQD+qWeubABwlWxuoddFvxvAX4loBTQn41YhRGUmBiwDb7NFkhdsXDq9OSoPmE5hju7fTbMfKcMwTEvhtcHFMgDjbJvNDS62AzjNx3G5IiN0sxiHkuUwKpCCbXZe3CJ0hmGYXCHnVorG407LJRxK79tQVWNkQWcYJtfJPUGXlospKg+nGaHLaowxU4huPgUX52IYJhfJOUG3py0CcC+b6IKsnW4u1mWO0N1qwTAMw2QzOSfoqpWi+WlaLgcORwAAZt02n09G6Gy9MAyTS+RcxyL7SlEA+MH43rj/nbVpnysmBO55YxWCQbJ0PPrHgk14d9VO7DpY7/jMjgOHccmTC3DJsb1x4HAE7YvycPxRHTGku2MdFcMwTLOSc4IeE07LpaxNHm49YxDum7MmrXPF4wJPf7JRuU8l5gDw4uKtqNhbi9+9bb1WxazpALQniMc/+hbnjeqB7mWFaY2HYRimKeSc5aLKcgGALiX5AIA+HdoAAH44sW/Kc8UUK0Z7tVeLsBACd72+Cv9Zkrw+zPYDdfj9nLU49YGPUl6fYRjGT3IuQletFAWA80f3QNuCMCoqa3DvW6shXNtYJFDVe8kPBZXH/ujvi/H+6t0pz1lbrxWZrHGpyc4wDJMpfGlwoR9zEhEt0xtcZCw8VaUt6tfHqUO6GJOb9uC7OD+EV26YaNmmEnS3RUpexBzQCoM1BSEEnvlkI95ZubNJ52EY5sjDa4QuG1xcoC//b2PeSURlAB4DcIYQYjMRdVadxA9UK0UtY3H5HBEwqleZZZuqEUa4kWUEKg/Vo2NxPmoaEmXgY3GRtJm1neG/fgfHH9UB76zUJmilL88wDOOFlOplanDxDKA1uBBC7Lcd9n1o1RY368d4C2cbgZuHnhiv+nNSWJ+4dKyx7VCdtQfHOzediHAwvZx2yeodWjEwc/u7g3p6pFeq66KGmDMMw6SLXw0ujgbQjog+JKIlRHS56kR+Nrhwi3zJOM5qp8hiXuac9eq6hOB2LM7HwK5tHZUbvSIF3Wy5VNXaGzulR/nMN/Hl5qomnYNhmCMHvxpchACMBTAdWrOLO4joaPuJMtHgwg65hOjSWzdbKofqExF6QVjb3tjKjbL5tDlCP5BGhB53qd/++vLt2F1dh301Tbs5MAzT+vGrwcVWAO8IIWr0srkfQ2sm7TtuWS52hAD+Z3I/4728AZgtlWqT5ZKnR+6hRlouOw/WAdD6mUpeXbbd8+dVBcMA7QZ27L1zMebu9xo1rpbg/72wDC98sbmlh8EwRxx+Nbh4FcAkIgoRURsA4wGs9nWkxni0f1N56AICV0woN7ZLy8VcmXGvKeqVkXko0LgIPa7rca1povWvn1VYInYA2FOtXrBkvhGY8dJKL9uYvXQbbn1pRUsPg2GOOLyql2xw8RW0nqG/JaJrTU0uVgOYA+ArAIsAPC2E+DoTAzZWirqMXMq8EEBBOJFTLi0Xs6ViFlcZoeeFrDeK4T1KPY0rqiv6btsK00g8EXkv27Ifx9z7PmYv3er4vFu6Yy4KOsMwLYMvDS70Y+4HcL9P43IlleVy1ojueOnLbbh28lGWCdCgwkM3E3aJ0KW3ngpZcXfe2t3oVlqAHQc0C6Y+EgcKtH1r9InT+esqUVFZi6smlqOsjdYPtU6RQgmwoDMM451Ws/Rf0q4oD6/cMBG92rexCLo83i0t0c05l1F+x+I8nD+6R8pxVVbX4+gubY3tt7y43Jh8lU8JL3+5DQ/PXYdZpnowrhG6ojwBwzCMitwT9BRpi2bMKYiJSVH1tyxs/0oStg0lbXUXiwsIIVAbiaG0MGxs/3DtHny2vlI/g/MzEtUiJ8A9+4VhGMZODgp68pWibgRdSgJIYkafUesBUtADhKQ56qt2HMRD76+DELAIOgAU5Yf0c1gH3SYv4fG7CXq0BQW9tiFqSe1kGCa7yT1BT2G52Hn5+uMtx7cvzlMeJyc17YJfoNs2RECei10jI/eH564D4BR0ebOwP1WYJ21jLm3v7Fkyzcmx987FsF+902LXZxgmPXJP0NOwXICEkMvji/NDqJg1HScM6Agg4alHonqEDrcInVwjdLuNYxf0A4cjuPYfSyxpkuZzA+5euSpCPlgXwQPvrkU0wx2VODpnmNwi5wRd1eAiGfIwe0QvRbhzWy0FRfYRjds0slC3RQjui47yQskF/flFmzFn5U488dG3ynNr11ULumqy9L631+CRD9aj/y/fxtn/94nyc2YmzvoAf/pgXcrjGIbJbXJO0L2uFJXEFD1IgURk3rVUF3R9paZbhE5ExuIkO/YIvaTQmg0qnw4abKtBq2oaUD7zTXz8zR5Xr7xGESWbV7iu2HbANeVRsm3/Yfzh3W+SHsMwTO6Tc4Iu0hR0Gfnag2tpn8hOR7IhtN35KCnQxJnIKqRm7E2q5SSocS1d0O1++MfrtOyX5xdtdhQTk5htDzlGuz1z4Z8/V36WYZgjC98aXOjHHUNEUSK6wN9hJpC2secI3WVCUvrPXUpslotNV6U4EwFrd1Yrr2G3XOxdj4L6YqUGm+e9rarWGIPbAiKz737Tv5dpY7Qd+9XWA8rPpmLEr9/BuHvew2vLvdecYRgme/EaocsGF4OgFd1y1GkhoiCA+wC869/wnMRTLP13Hq/9a6/CKIthSQ89YtghVrEsNkXbU4eo+3bYFytFbUa8W7bjQT3i79Q2HxV7a5XHmIX+zRU7HNskjclXP1gXReWhBvzk+aVJj0tntao97VO1/x+fO2vcMAzTdPxqcAFo9V5eApCx5hZA+mmL8gZg97+lny0tl8uPL9ePt36+Xyet9Pv2/XX40Qn9sPaeMxzXCNruLhP6dbC8TyWIW/bV4pG53ictVfZMXTSG7fsP4+tt1mg9lcB6IZVHb71e8v3vrdqFO15difvmrEl+IMMwaeNLgwsi6gHgfACPJztRczS4sONmucgIvbQwjA2/PRM/P10rJmkXQLmMPxYXICJlE2nzqYd0KwERYUzvRLs7t9K4knT7h6ruD4fqojjh9/Nwli3rxY9aMIcjMXy4djemPzJfOUlrHVvy68k5gf1NbP7BMIwTvxpcPATgViFEUuXyo8FFj3aF+M6YHigpCKc+GAkDxSnoWtRZlB9CIECGJWOXo3AwgPu+OxwvXaecNgBgfVqQqY0vXz8Rn808WbtWJLmgV9Wm16ouosg/P/a3c5Xi7cdK07pIDP9cuBkrtx/EG18l99u9Xs6tEUlDNI6hd87Bq8u2pTtMhjni8avBxTgA/yaiCgAXAHiMiM7zbZQmRvUqwwMXjjLSDVNx/FEd8P3xvTHru8Mt26XlUmzLSPnRCf1g56JjemNsn/au1zBrk/nGIV/Lm4cfnPV/8zFfz44BgKmDuziOMV8vVcTshbpIDO31qpCpbj6prid3uz1fHTgcQU1DDHe9bi+5zzBMKnxpcCGE6CuEKBdClEMT/OuFEK/4PdjGEA4G8Nvzh6NbaaFlu7RBzPVUAGBi/46omDU9rWuYo01zAS8ZuaeyXNLh620HLe8L85wWkLn1nR+WS10kbtSaSbU6NZWgx1MoupxgtufsMwyTGk/10JFocJEHYAOAq0zNLZ5I+sksRQpGftgpiOlidnPM9ouM0Ne4pDv6QRvF+A/URozsHa+CLoRwtUH210aM1MaIS80ZSarLyd2kUPRYXBht++wpngzDpMa3BhemY69s4piahRum9Mftr3yNDkXqYl3ThnVNKV4SszSdeHRibsDrxG1TaGyEbp/8jcaFER3PX7fHkq3y188qUp5P0hSL59lPN+KeN7WMWBZ0hkkfrxF6q+PS4/rg0uP6uO5//NKxns8lo/KbTz0a100+ytjeHIJeoIrQPQi6/WYVjQnIU132zCLLvnxT16aIvdiNjeTT4jBCdNWPZpfeaBtInf7IMIyTnFv6n41IQe/VvtBSM8at9osXfn32ENx93rCUx9nnAABruQC3Ko72xU/JhDps+p6iMYF4XLjeKLx66KofDYs4wzQNFnQf6NRWW5xkX+zkdTWrivbF+ejbwZLur7SHChURukXQ04jQ3TCfLxqL47K/LMRRt72FK59d5Ji89Gq5qDx0hmGaBgt6E/j9BSPwxo2TjGjTPqlobzidDkEiS7nesX3aoabBuainQBGh19RH8fE3e7Bww16LoO88kLA07LnsybJXzEXJInGBT9fvBaC119tQechyrNdJ0XT3MQyTmiPWQ/eDcX3aoV+nYsMqsPvCTbHQgwFrjZg2eUHUKRYoqbJcDtVFcflfNB/8g59NNrYf97u5eOm641EfiTnEM1lqpYzQ2+QFHZ2V7CtnU9dy0f413+sisThCAWLLhWGaCEfoTUBOehq+sM1GcEsDBIAZk/riJyf3d90fILJE+KrJT0Cd5XLQFFH/0VYHffmW/fj+0wvxg6cXWraf8Pt52HuoXnkNKehF+SGH1263dFJF6AlLRvvZ7DxQhwG/fBsvfLHFUYueYZj0YEFvBHl6+UTpmbtF6CqevGwsKmZNx+1nDcHNpw10PS4YsFouqslPQC3o5jRDWaFR8od317pec/v+Ost7+f0cqktE6HavPRKL4+Nv9mDHgcMAnB565aF6nPfop9i+/7BlOxHw+vLtWLalyhgnR+gM0zRY0BuBTONzROgeslrs3Y3cCATIcqyboNuba6RC1dJOYhdj+YSwt6YBeaEAwsGAMzsmFsflf1lktMKzn+PlL7di2Zb9eOaTjQCA2Uu1Gi1LKqpw4/NLcetLKwA4SzAwDJM+vjS4IKIfENFXRLSCiD4jopGZGW52YG4cDZhrrqf+rFtfUok8R5DIUkagMKwWPHtqZJ8ObVIPwgW7GFsW9witrIE9O0a+rzykVU+0R9nF+VoRtUN1UQghsGSTFpFLG0fmzLOgM0zT8fpXJBtcXKAv/7erxkYAk4UQVUQ0DcCTAMb7OM6sokCP0GURLGF46KlJlfkSIEJMCATIW4TeodiayhhqwkxsslWgDbE4wsGA45hUaYtt9RZ+h+qjSf314oJQo5p0MAyTwJcGF0KIz4QQVfrbBQB6+j3QbOJ/de9b5p9LGfLSdMPe3ciO1ONAwBrNq7xyAGjXJg+zrz/edP7Gu2ipiogFA+RId7Q3v7BrsrwRVddHlWV/JUI40xb9aM7BMEcSvjS4sHE1gLdVO/xocJENnDuqBypmTUebPC36nDGpLwBglKmphZ2juxQDSDSnlnx3TE9LCVxpYZQV5llKB7hF6KFAwOiL2lQ27a3F7uo61/3hIDkmRQ+bBH3V9oOWTJmtVbXYp/dE/fzbSrz8ZaLGuf3ep2pJ53aDqTxUj/KZb+Lfiza7fzMMcwTixXKRDS5uFEIsJKKHoTW4uMN+IBFNgSbok1QnEkI8Cc2Owbhx41pN+HV8GiV37ZbIHy/UphvKZ75p2d6vU5FF5FQrQgEgGKSUvrxXbpu9ArfNdt8fCgSwqGKfZZt5jGc+Mt+yb9J984zXkZjAbbNXGO/tgl4XjTmabdfUR5Xpmlv2af1Xn1+0GRcf2xsAsH73IfRsV+ia3skwRwJ+NbgAEY0A8DSAc4UQe/0bYutAugepxLej7okXhIOeLJdQgNJakXrOyO6ejzXz2cyTEQqSw0M/nEa/UTP20jGHG2KOPHS3jBz55CLr1Bysi2DqAx9h5ktfNWosDNNaSBmhCyF2EtEWIhoohFgLRYMLIuoN4GUAlwkhvlGdh9FIVcPknZtONETSOinqkuUSoLSqOjYmmn/u6vHoXlao9OfTaSBtxn5jqFPYK4cU/UsjsbgxVyEteZknv2DDPsfxDHMk4VeDizsBdIDWeg4AokIIe/10BqlTGzsU5xuvQx489CCRZaJ1RM/SpA010s2CuWJCH0wa0NH1syrv2wv2eud1DTFHyqO9IfXWqlpMum+eMWcR08N8t0bgDHOk4UuDCyHEDAAzfBxXq+Hpy8ehc0k+fvaf5Wl/VtWfVDKyZyn+Z/JRCNgi9LvOHYZoXFgmIM2M6d0O/1m81fMYzE2mVRF6Yy0Xe2S/bnc1lmyusmw7WGftX7p+t1YI7GV9cZIUcpk9w4LOHOnwStEMM3VIF4zoWWZ44OlIjnnlaZHNculaWoAzh3cDAEfNl7F92rme8ztjeuK35w933W/H7Iyoni4aK+j2DJaq2ojDhqmqsQq6bINXH5H5/9ZzsaAzRzos6M3EYz8Yg5+c3B/9Oxcr9181sRzThnV1/fywHiWW92Z7wi5kyXz6UICMFEovmBf7qDJt/PLQVfxn8RbsqU6kQcqnDmnXyElROYbG6vm6XdWYu3pX4z7MMFkEC3oz0bNdG9x82kDXei+/Onto0rZ3RITThiTy1Ru7qDIQIEcufDLMlov08eWCKgBYunm/4zN+sXDjPlz69EIs27If/1iwydguc/XlTUEVoQsh8M+Fm1CrqCFv59QHP8bVf1uc9vj21zY4bKGW4tlPN6J85puOeQfmyIIFPYeY9d0RGNJNi9STraI03zMGdW2LX0wbZNmfbGJ04+/OtLw3L+Uv1G2felNUnmwC1g82VB7Cdx77FHe88rVjX9wh6In/zl9u3o9fzv4aM19a4ficX4y66z2Mvuu9jJ0/HZ76eAMAoKq2oYVHwrQkLOg5RPuiPNxyulZ2wGuAHg4G0K9TsWObmYuP6YXffWc45v98CogIj1wy2thntkamDu4MADj+qI6NGH3jiAs4FhxJpOUibzDmb0vW2Xlt+XZL02y/8WId+cn2/YeVUbj8WZifUnYdrMMXFZzKeSTBgp5jyOg7We9Oc/wdIKCkwDqhas9Fn3FCX1xybG/0aq/VXDtnZHc8dNEoANYm0+PK22P9vdNwTN/2AICJ/Ts09tvwTFwI9CgrVO5zWC6mR5PvP5Vo4CGzY1oDx8/6AJc8tcCxXWaBxuICz3yyEet2VWPaw/PxvSc+b+YRMi0JC3qOQbaSvepjrO/bFoQt7/NsEXqRonStjOLtFRBDwYAx+di9VC206WIfjxlV0S6JFHQ5Kbp86wH87q3Vjtx42Ut13trdeF3PlFFfK3W0vae6Pmn/1ebgq60HLO///NG3qNRr6DRE47j7jVU499FPjTo6zJEDC3qOIbU6nUqEbW0RunwsLwgH8PTl49BNIcxSY1WWgvy8l/rvkr/98FjXfT3bJb8xuH2rVbWalWJOgfzzxxuwt8baSk92U7rq2S9w4/NL8c+FmxwdlAB3+6QuEsPUBz7C3NW7cMy97+OOV1c6jqmorEl6s8gkv3t7jfFa/iySNTJhWi9+NbggInqEiNbrjS4ctV4Yf7C3vVNhSVskci1YVVaYh6mmzBnVdVTWjnxKIJDnjkmTj+5kvL5iQh/Lvt4pmnIk86nvfXMV7n/H2lbPnL+eFwpYUh8B4Jezv8bxsz5wnMvcvCMSixupjGt2VmP97kO4Uxfyt7/e4fjsuY9+ihufX9riNd0bm0bKtA68RuiywcUgACMBrLbtnwZggP51DYDHfRshY0HaHek0VJYNOSQysm6T716Z0CiApRAoc5KMW0mCZHx3rLVcfidTuQMV5jF0L7WWCn5q/kbHpKc5Qi8MB13L8G6tqrW8N5cOfmTuOlz9t8WYv24Pdh/Utncp0evfK370cgyZzDLxcrNIVdOead340uACwLkA/i40FgAoI6Juvo/2CGS8PgEpGdOnHaYM7IRfnz3Usv28Ud2NDBi7UNsj9K4lBbhp6gA8e+UxrtcNGBUNFftMXotbWd9khAIBSy57qhWe5qeEdkV5SY7UkKL6tx8ei3AwgIZYXOl7r95hTbmcfP+HxuuNlTUAgH01DdilR/id2xY4xmNnbwZ966gHQVdF6C391MA0H341uOgBYIvp/VZ9G9NEnpsxHqvvOsN4XxAO4tmrjsWALm0txz108WjcMKU/AODMYd0sK1LtaYpEhJumHo0+Hdz7lMiMEZUYSD0jspb1dVsFaycUJLz90xPw4EUjMfdnk42bhxtmAW3vQdCr9eqLQ7uXIC9IiMbi2Frl9Mw37a1xbJNzE/KKRIT9ukiXFuqTywp9lLn9lSZ7Z/bSrcqJyd0H64y0Sjtzvt6J/S5Rvuppyf77qYs4b1zRuMAjc9c5rCem9eFF0GWDi8eFEKMB1EBrcGP8L/oAACAASURBVJE2raVjUXMSDgZca6G7EQgQrjmhX5Oum8xykRAlyvo+cOFIo4ytl3N3LM7H+aN74qhOxSmX7JuDay+CLicE2+QFEQ4FEIkJnPSHDx3HbdlXi8F3zLFsk6InhT1A1tRNQJ11I3PlK3UBrzxUj//3wnL88K9fOI499rdzcd6jnzkmtncfrMO1zy3B9f/8Uvl9Re1F5AHU2FbCqm4UizbuwwPvfYOfv5h+gTgmt/CrwcU2AL1M73vq2ywIIZ4UQowTQozr1KmTfTfjIwX6TSDSSE9VCme/Ts4o3uzfS1+5T4ciVHtcBh+2NeQIpkiXMYuUl3Z7tQ0xEAEFoSBCAVLWVQeATftqHcXFDuo3Jam1ASJHVUeV5SJb88lFTjJy/nqbNcVQivjqHQfx7ipr/Rjpf2/aa/X2Jaqba8TmidUrInT5Paqid6Z1kVLQhRA7AWwhooH6JkeDCwCvAbhcz3Y5DsABIYQzFYBpNop1H72xWQ+Du5XguavH446zhiQ97v4LRmLmtEEY07sMNSlS5aTfbtdvt/o2EvNE3+he7n1bJY/MXQchtCeVcDCAVdsPArBm2gDAZoVwyhz2uDlClwuYTIJuF1ejYFhc4Jtd1Vi8SSsFHI0LVNU0YPPeWvzniy34dk/C5pGt9CTyx2CO3OsiMcO2sYs3AMfcQJ0iQpd14+0/5kgsjvKZb+Kh95vWk+Yfn1cYP2OmZfGrwcVbAM4EsB5ALYCrMjBWJg1kud2mZD3IxhbuENoV5eHayUcBAK48vhx//awCt5w+0JFKCAC/+85w3PTCMsfEZiCFoDeYvodUKY52quui2KbnnPftWISPvklYfVuqnIIui3lJvX5l6XaUtQlbxiFEIlq3E40LnPbgx5ZtJ/5+Hqr1pwRzmqdbyqdZtn/w9EIs2VSFilnT1RF6PHWELm8E9p+ztKaemb8RN009WjkWL8i8fK99dZnM4VeDCwHgBh/HxTSRYn0xUSbykt2SPH519hDcedYQBAKkFPTzRvfAeaObNlfuNe9dctCU0jisR6llnyrilSIno+Q5K3ca++TNUcBd0FWiW22yfMw3WPviH2mRmy2dJXqkH4sLpYces1suighdjtV+35S2UKpJ6XTYXV2Hu15fhfu+O0K5ApnJLLxStJVSnJ9BQdf/VVknjRGHdHLq80PpTRDLVL+nLh+H3u1TR/cJQXfuM+YjRMIzd7ueF+z+vRRs1bX31za4ROhWkX/gPad94tbRyY9OT/aJ3Qff+wZvfLUDryxTd8xiMgsLeitFRkeN7SjkF8eUu3dPagxuWS6lhWHHClQgETEX5QctvVfdSFguTvFsMHnoZiE1pw7GFFG0G05Bt6ZMmqk81KC8WdhFXnU/cbNc5NNCKssrGfYxBZKkuzKZh5+JWikyQpd/V9NHdLPYD83FC9dMSBl/qyJSIvV2t5WpcSGUJQ5k1FsYDip7oto5HInhmr8vxry1zrRaaWfEhbBE6GZxTydCr7NZLvKc5u+7tDCMA4cjqDxUb1mMZVzbQ6EwOXFql+0GI0KHfl2BPYfqjQVUXrDfULykuzKZgyP0Vor0muVin0e/Pwb/uHq8r9fwEtfZm1h7xS2VkUhblHT79MGW7dV1UYtgy9K+UlcK84KuddXt57GnE0oaTB66WdDNr+2edjLsEXpCBJ1dog4ejijF2836sRyjn9eeTWRYLvr2p+ZvwLH3zkVFpXPBlRvmMe08UGdYfH7o+cG6iHLxF+MOC3orhYjw0nUT8O9rjvP/5GlUemwsybz4wd1KMOOEfrjjrCFGKmRZm7ClzvsjF4+2fKYwHExapve7Y3qCKFFqV4Uly8UclZvFPS0P3SrQ8mnC0phb/zcSd6ZKer1ewnKxbpffj/xZywygxZuqcMt/l7uuZjVjHtNxv5uL/yzeqn8PAnWRGMpnvoknPvo25XlU/O6t1Zh8/4fYsKf11LPPNCzorZixfdqjY4rCV02hCdZrSrwE9VdP6ovVd5+B1388Ca//eJIlQrfbL4VhbdWoG+EgoUvbAqza4Z5PbU6fdLNc5ny9E16x122X4mieaJRRdSQaV4q3l9rs8hi7V26fFJX7f/Xq1/jvkq2Yu3p3ynOrMoUA7XuRBcue+WRjyvNIhBC4/ZUVWLn9AD5ZXwkAltx9JjnsoTNZid1yeerycRjXRz3BOrynlo5ojsDt6Y354SCQJOIMBAh9OxZh4ca9rseYUw7NVoM5Sl27y3uPVXsGkhRH9cRmXBmhe/GqpVduW6Cr7PQEJEodyK2frKvEnkN1OH+0tUrmcws2oW9HdT2guDDX7k85RIM91fV4bsFmvLNyFzroE+DpTDQf6bCgM2nj5e/zXzPGY5/HUrKqZh32aLJbaUHKSovmLJaQzV4pDAeTNgUJBQh9OxXh8w3ugm6O0F8zNbPwMjFpJz8USOKhO4nE4kq/3LywKD8UUC4kk9ucHro1D91ea18efukzWtWPc0b2sMyH3K5o3C2JC2FcL51mLBIhEuNNx8Y60mHLhUkb4w8+ybTo8f074qwR3Rt9DbuHbq/priJZEbNwkCyTooO6WqtVBojQzyXalNSbhPuDNQk7wsvEpJ2SwjBq6qP4yycbsb+2ATsOHDaE02q56NeOxpVrCszRq9tCngaX9MQGW4SeKD2gHvM3pqcPe1qiatGSXF+QrNywA3K+5IwZ73iK0ImoAkA1gBiAqBBinG1/KYDnAPTWz/kHIcSz/g6Vaa2o/lztHrqXBUWqtMVbzxiEp+ZvABFZPPbfnDMUFz2ZaLYcChDKk5QTBqwReq1p9ee7q7z75pLSwjDW7KzGXW+swrIt+xOleaEJ6q6Ddbjoz58bZX/vedPeU0bD7GG3yQtin8JuljeCVJOihkVi+40ESLNQvtlVjcHdSgAAo+9+z3KMXbNjQhgrX1W/XyGEUW9HjTBuEo25YR6ppBOhTxFCjLKLuc4NAFYJIUYCOAnAH/W6LwzTKOzRZL6XCF0h6NeddBS+vONUAIma5UCi7K8kGCT0Mq0kTdXn1FyI7LdvrUlypJoi09PEgcMRy7J+AWD20m2ocKm6aCZqE3QV5gVEkVjceAKI2PLQU7U3/Om/l6FKLxRm7xJlJx4XRmSuOt8tL36Ffre9pfhc4rX8L8ARunf8slwEgLakmV7FAPYB8FYcm8k5pCD4leWiXlhkt1xSR+ip6sabzxmyrRoNElmaab9/8+Sk56pxKcnrGdNYwkGyRNqqao52rv/nEgDWGumXHudcKQskFkRV1TZgwC/fxj8WbAKQiNB37K/D/e+sMSJpu0Vifvfcgk1Y52HiNy4SQqyyXF5cslX5uZjpJiAtvcZ46B99swd/fNdZT6i141XQBYB3iWgJEV2j2P8nAIMBbAewAsBPhRCO2RlucNE6COohXcieNuEj3cusqxULPFgu6bTDs68aDQXIqFAJpC4C1tSJuqHdS4zXW6sOW9IPk1VzlLy1YieEacXqR7echKNtXawksgLjt3o+96vLtAldmf2yt6YBj8771mi7Z//OzHr8x/e+wam2apLKa0ZjeE6/cSSz0O0TpmZvPhGhpz/pfMVfFuH/Plif9udyHa9/kZOEEGOgNYO+gYhOtO0/HcAyAN0BjALwJyIqsR3DDS5aCd8b2xNXTSzHTacO8OV8quJcT1w6Fr8+O1GL3UsdFi9RvMS+yCgQIEsv1lQ12pvCBWN7ootpef2andVYuHGf8V5A4ODh1E8AW/YdNqLgUDDgehOSlsu+Q5pdUlZoLQcs2XFA8+ulxl77nLpzkheemr8Rf/54g34+d0VvsN24zDdKUmwDtKej9bu9LTY6VB/FtIfnOxqNtFY8CboQYpv+724AswEcazvkKgAv602i1wPYCGCQnwNlsoeCcBC/OnsoSgrCqQ/2gOrvvVtpAS4Z39t470Vg0xF0u+WSFwp4qvWSLh2LnVNJBKAwz3qtHaYVqnEB7K1J3f/zwOGIsagpFCDX8UvLRfr+cgLW/hSQqY5GyR5mlm85gEfnJSJpi9VE6rowV/31C0x94CNP117w7V6s3nFQWYWyNZLyfzARFRFRW/kawGkA7Amom6F1MgIRdQEwEFojDIZpFESUdKm+Ci+9V5+8bCze/ukJDkEvyvN3SUaPskLcc94wPHjRKMsEqCSZPRQ3rbJMRiSeyE1PLuhWoZbyaI/QVZTPfDPlMak4HInhyY/Vy/8v/PPnuP+dtUYUL/32vTUNWL5lPwBnhL5If5rxUtFR5vqnY8cBWtXNp+dvyLmqkV7+YroA+ISIlgNYBOBNIcQcIrpWdi0CcDeA44loBYC5AG4VQlRmZshMa8PtTyZd20P+0SYrG3Da0K4Y3K3EkUWTbiPuVPQoK8Slx/XBCQM6YeVdZ1jsECJgi56OKLny+HLjdTQusM22X4W5HEAoEHDcpCR7D1kXeMner3a7I5PYM4Hsv1p500lnNazbPIbZ4pHlFdx+v0IInP/Yp3jPVpDt4ffX4Z43V+ONFbnVSTNlWCKE2ABgpGK7uWPRdmiRO8O0GHLxkZfqjvaIzS3lz56B4hV7Zod9TN8b2xNfVOzDXecMQ2lhGL07tMFfP6sw9q/z4BFH48KYTA0F3Z9oZAs+yfurd6MhGm9WQbcTClh/rvXRONbsrMZ5j37qONYtD91N6M3bZX17twj9wOEIlm7ej5/9Zxm++vXplvEAWimCXIJXijItjtS+e84b1qTzFIaDOGVQZzx9xTEpjy3KD2HBL07BiXrjaFUWzcLbTkmZvuhGKkEf0KUtZl8/EcN7lqbdJ1USicUN2yQctEbonRW1080M/dUc/PmjlnNF7RlS9dEY3v5aHQ27ZbmoWvIJISzlj+W8gdsNe6+eV28vKyFTWA/V5Vb2NddyYbKKjsV5GN+3Q6M+S0R45srUYi7pWlpgiIVcuPTfaycYlk2XkoKkJWTbF+VhX426Xo09eDQLul/Vh6MxgUP1UeSHAo5J3fZFedidJLpszFOHH0y+fx6um3yUdvMxTROomltL3KyV15fvwCXH9rJYc/PW7sb1/0xk51TrglwQDqIuEoMQmv2ydHMV2rXJM+yosjZ5qK6LoDAcRCgYMARd2lNNZcu+WsvCtUzBETqTVSy+/VQ8+oMxzXY9KSTSrjimvD3G9mlv7E9WcqBHmbaa9LxR3fGXK60LqO2peiEPNtAfvudwNpMSjcdxqD5qdKcKm6JeVXcjSSbLHicjFhfYtLcWM19e4fh5bK06jHlr1OV63ayV22avwNQHPrLUS99XYxXgQ/Xa+1CAcMLv52HwnXMAAOc/9hlO+sOHqDyk3fTKCsMY/ut3cdvsFQCA4vyw/vnkEfqQO+fg8Q+T13v/7NtKnPD7eXhlaeb7rLKgM1lDSwiN9JHzU2RBdC9N5I0P0euZdCnRtg3pXoKTB3WxHG+vQ++lb+cFY3tacu/Ni49UVNdF8c+Fm40J13AocY0OSSpTdspgjfxkmJ9m7NUwL3lqAb7ZpZ43SLaI69s9NZjx98XGe/t6BXnDjsaF0g+XY5IpnHLRlbzhVKewXGobYrhvjnvph42VNViwQcvKMa81yBRsuTBZgPsf7MMXjzIi0EwgPehkKZLzfz4FJQVhjLzrXQDAAxeNxIY9NZi/TlvtLKO5vGDAuEH88UJrtG2OSJNpu7yxdC0pSNky75YXvwIAbNdz2M2+dFkbd0FX+cmyd2kmMbeT8/LEIklVBsFs19hTN+Xk5gKXsshSyHcd1H6GnUu0m50sQVDT0DQPfcofPjReP79oM66bfFSj50y8wBE6kzWoyvGeO6oHThncRXG0P9xy+kCUFoZdGzUAQK/2bVDaJrGIqqwwD2cO72ZEb0X6CtOld2pFwEoLww5BDXpY6QokMnXyQgHDa09VhkBijk7NdWnMzP3ZZGUWUJ8Mioxkg6nzkFuKpQo5+bnjwGFlXrw5W8f+JCQrTbpFx/JmIRd2yScruT2Zt9+YOu8XP/l52p9JB47QmRZHil9Rvr+54F44ZXAXLP9Vehm3MhCWBbrkE0RRfghv3DjJiPLMuDW9tiOzbcJBMp5bOrXNN8roJsM8Oej2VNOvY5FyAVLv9m3w1dbMLo/fZKrtG06jDpAU19+8tkq537zi1b76VdXww4y0c2r1bBh585RZSvZJ8QOHI6ipj6J7WWGjGmFn+imIBZ1pcX56ygB0Ky3A2U1oiNGcSHHurk+KSi8dAIb1KFV+JuAxy0V2IOrTochIqetY7E3QzbiVQSAiZYR+1ohueOOrzC6ieXReYvLQy1oBicxDn7NSXXc+4tIaEHC2+bNjt3Pke/mvuRzCul3VmPbwfETjAhWzpqfXuEMnnfIUjcHTbZKIKohoBREtI6LFLsecpO9fSUTeCi0wDLT/5JdPKE/S7CA7kHaIfKy/ffoQPHvVMa4ibqZbaUHKY4CElz9lYCdD+ctNdsgVE5wlch+40Jkdc8KAjjh3lPoGqfKvB3ZNPgHrN+nUzYnFRdLqkzIFUwiB//3vcsu+OkWE/uynG43X9kVLiZK/8vPaDWHl9gM49cGPLRO0OSvoOq4NLoioDMBjAM4RQgwF8D2/Bsgw2YJcbSgnzArzgpgysLOnz/7fJWMwpndZyuNOH9oF/5oxHpce18ewXM4Y1hXj+7bHE5eOwRnDujk+c0x5e8e2fp2K8fDFo5XXMGeY9O9cjK9/c7qnFn9+4qVhiSQaF6itd4+0G2Ja+mZVbcRhg9QrIvTfvJ6wbmLCRdBtHvrug84MGfO6pm/3HMJLthrvUcVNyO8SE3b8+i1+H1q1xc2AUZWRYVoVd52rLdNvTJXJ9kV5uOiYXgCSZ7kQEY7v39HihxeEg3jhfyYoxRxIHu3O+9+TMO9/T7JsM9sdecEAivNDnurN+wWRt/r2klhcpMw2Gfard3DrS185tqeyXOwLh2QEHjM8dE2U7Q29AWuEfsZDH+NntqcDVWmFdIuEpYtXD102uBAA/iyEeNK2/2gAYSL6EEBbAA8LIf5uP4neHOMaAOjdu7d9N8NkNWeP7I6zRzafzy/1wpxSqaodn8yPVmXvmLNh5NykmxXQNj+Eao/dmYi8rYIVAvjcJY1QRTQe99Qhyl5gC0hdEvjZTyss7+0eupwUtd8YhBCW6N5s+8heqaprZ1rQ/WpwEQIwFsB0aM0u7iCio+0n4QYXDOMdKd72RTh2vDT/cKOdnmHklho5oEsx7j53qKdzeU2vTJd3Vu7y1CVJRV2S0g0qonrp4jU7tTZ70nKxi/PBuihiivIJ9765Gv1uewvxuFCWjUhnMrgx+NXgYiuAd4QQNXrZ3I+hqNDIMEz6pMrZTiX4dsyZHe31FaXJJqQvm1Du6bzp1q9vDlJZLnZicYFLnlyA15cn2vQJIRznGfmbd3Hnaysdn3/6E23Ctbo+qsxh71bmbXK8saS0XPSmFgEhRLWpwcVdtsNehdZ2LgQgD8B4AA/6PViGaQ14TY6Qx6XK2TZnrbRrE0a5wmb5bObJpgk/mI53X1HqlbxQAA3RuLbKNcuqE6bbhakhGjd6q0qicaH00KXom5G2U1VNgyMHvqQglPGGGV489C4AZuuTNCEA/5INLgCtLroQYjURzQHwFYA4gKeFEPauRgzDpIEUdPMkaq92zhWdZkFfeqd6kZTMmQesmR0/OrGf8vgORXnYW9PgqcmIzMvPxgg9XVQTmdGYUGbLqAgHtPIPVbUNjlWr+eFgk5uLpyLlb0AIsUEIMVL/GiqEuFff/oStycX9QoghQohhQoiHMjlohsllmlKErFf7Nlh8+1TLtnR9WRmpP3DhSKNipB2veeKv/3iScYPIlIduxkvqZ1NQteWLxOPKfHYV0h6rqnVG6PmhQMq6NE0l92+pDNNKcfvT71icj2V63Rgg/VZ9UlSSlQaWhcFSnVlAGDZCqmJiTWVi/w5JyrhpjO3TrknXUE1kRqJxz168vLlW1UQcKZH5oUDLR+gMw7QMsviTSq+TVVNMRTSeOqKWmTPy2qN6uUfGxvkynWMdCCjnE8xPGaN6leHoLsWNvoYqQo/GhdGbNBVS0A/WRYzSDZL8UJAjdIY5UjltaFcAyZtVNAa3iPpvP0wkr43q1Q6d2ubj5lMHAgBeuWEibj5Vy0T+ySkDMFxR7iA/wx56KEAoUKy0vPu8oZZj7O3t0kGlt5GYd8tF/mxr6qOoMgk6ERAOBfDBmt34zevO7Bi/YEFnmGZiRE8tyj15kLdyATedMgBLbp+Kzm39TXWT5WjtEfrkozvhuH5aGYH2RWF88cupmHBUoh3g9BHaStWzRyRWrJozdmojmc1wCQbI0mhEUpQXshxjT/O87Lg+mDasa6Ove7ghhm1VtZ6OlcJfXR+1NPQIBwJGa0P7YiY/YUFnmGZicLcSrL3nDNcl/HYCAUKHDHQXklGoyiKRk6Eqf/2oTsWomDUdA7q0VdpAXm2JZAzrUYJBXdsa7289Y5DxukNxHm6bPhi3nD4Qn8082dheZCoVHAoQhna3Pj2EgtSkwm83/2c5vty839Ox0rKpsQl6MEDYsb/OeL8oQ92LWNAZphlJNhHZXMQ8eOipJjhl+qS5+1FT3eHvjOmB126YhFdumGhsM/vht505GCUFYdwwpb8lDdMs6MFAAL8+ZwjuOCvRyq9NXjCtDkl2VmxLv078gcNRS+3zUIAs7+etzUy5K66HzjBHGMkE3agfk0LQZ313OM4a0Q0DurR1PSYUoLSyOjoW5yMQIARFQnzlKti8UABtXYqimW8qoSAhPxTExP4Jq6i0MGyJjpsD+6KjUJAsi5O8NjxJF47QGeYII5YkzdDYl2KCs21BGNOGW60jeTM4e2R3tC0IpV0qVmbWmMVObkvW7s08VhmJD+pags76ZHJpYbjFa+0HA9aUxUzVdPGtwYV+3DFEFCWiC/wbIsMwfhJNIuiyJGxjcsrlzeCKCX2w4tenY1L/jml9Xvr3ZvGV25KVSzBPgpqFUmbilBSEcYrHiWg/OKbcmQtvL6DWFAsoGb40uAAAIgoCuA/Au76MjGGYpHQtaVz2ixRtVXpfUwRddhWSpXgfvGgU5tx0Au4+dyguGtdL+ZlfnjkY4/TFQKrVqVL4VHouxdv8fZiFUn4mHAxg2vBu+NEJfdP6fhqLKivJnnqaqScGPy2XGwG8BICbWzBMM/DhLSdh1V2np/25H0/pD0DdSFpmaTSmVVpC0APGOQZ1LcFlE8otE5dmfnRiP2N1p732CWCO0J2SPqCzNmFqvi8FTTeFuG1hljzFtZOP8votufKTk/u77lPNTZw7qoflfUtH6LLBxRK9SYUFIuoB4HwAjyc7CRFdQ0SLiWjxnj170h8twzAGBeEg2uSln9fww0l9UTFrujIKrzykpdqlE/2/esNE3D59sFG7RJXJk6zlnJRqlcYZgq743HMzxuMvV46zXM8slDefejR6t2+DcbYWfR2K8tCng7PImVf6dy7GzacNRD9FVcvObfOV0bc9Qm9RDx2pG1w8BOBWIUTS5VTc4IJhspvd1Vo2SPc06naP7FWGGSf0M6J7lXgnS5GUqytViR8hY1LUua9jcT5OHtTFss0slCN6luHjn09BaaEzO6YR/Z0NjF6hivHOvmGi8sbUtsB6482UoHu6vZsbXBCRbHBhbiEyDsC/9SJBHQGcSURRIcQrPo+XYZgMIlupdWmEP2/30M0ky79PROhOkfNaklfWIU9mZQjL68YrupxUtl9p+vBu6FFWCFIofYlN0FvMciGiIiJqK19Da3BhqXUuhOgrhCgXQpQDeBHA9SzmDJN7vHTd8fj5GQM9l881I7PyVA2gk0boScLlVN2aJPJm4CXyJUo0+bh9+mDLvo9vmeL6OWlRyWwe+w1INsZQlZKx59C35KRoFwCfENFyAIsAvCkbXMgmFwzDtA7G9mmH609yn/BLRu/2mi+t6nF63ugejm2SRCMP90nRVMhPJivMpbpvnD7UWuOlc4l7qYWwLsJRF4tox4HD+nbn99GjrBCv/TixArbFInSvDS5Mx18phHgxE4NlGCZ7+e+1E/D3Hx6rFLT2RXmomDXdSFE0Y5QJVpwzVfs9ibykV2/arTRxQTiInu20sgKlhWH85JQBxj65alUKu91akdc2D+FqfQK6KD+EET3LjBWswSZUhEwGL/1nGMYXupQUpPTeI4pSAMmyXLxaLtpNRHiOfOU1VTefT2492ZgPCAUIj8xdB0Dr13rVxHKcNaK7/lnr52r14mRS6GdM6otbpw2CikxVGmZBZxim2Ygo6or7YblIHQ8muQGYJ0KNa5r2P3Hp2KTXJSLcNPVo1/NLQZdj6dGu0HEeed1MRehcy4VhmGYjomjCLIVWnYfuLeKWgX+yrBjzjUNe03wPOcOlZvqMSeoVpnJStLMtx/zCY7RVsafYUirNtPTCIoZhmCZTp+jZGU/4H459Mmo/a0TyGvIyB75LkklNM+eP7gnAmX2iYqJLTRo53KevGKefSzM8hnYvRcWs6eidZPGSKkXTD9hyYRim2aiLOCP07x/bG/9auBlTBqoXGy6/8zS0yfdWiqBbaWHqgwD8/PSBuPHk/q4lCcy4pRhKTSYQnrliHAZ0di8lbCdTEToLOsMwzUZdxBmhD+uhRbRulLZJHUVLvAg0QRNpeewHP5uMrVWHXY93q10uJz/jQuCUwe72ihnDQ/doJaULCzrDMM2GStDdkHntXnjysrH4dk+Np2Pt+tyvUzH6dSpWH4zEQiF7gTCj6JfnUSbIVIMLFnSGYZoNWVogFavvOkO54tKN04Y2vgl0KlwjdErdfMP1nC1puRBRBYBqADEAUXtNdCL6AYBboT3NVAO4Tgix3N+hMgxzpJButyMvNEZ4AXfxlVvT6LKXyK5p1EhSk06EPkUIUemybyOAyUKIKiKaBuBJAOObPDqGYRifSVdM5aSoXbcTgXtT22P7hy+WixDiM9PbBQB6+nFehmGYlsbNcgkYC8skjAAABxdJREFUlksjTpqhEN2XBhc2rgbwtmoHN7hgmCObMlPGSmEjuiI1hcbG0X5aLpnGa4Q+SQixjYg6A3iPiNYIIT62H0REU6AJ+iTVSYQQT0KzYzBu3Lgs+jEwDNMczPnpidh+4DAqq+sxqGtJi4xBVWIgGW6C/sNJfbF4UxX6d3bPkLHTlMYaXvCrwQWIaASApwFME0Ls9XugDMPkPl1LC9C1tHHNrZtKY8XUTdDPHN4taf58MlRNMPzAlwYXRNQbwMsALhNCfJOJgTIMwzSFET1LASSaS3slU8v0M4GXCL0LgNn6Y0oIwL9kgwtAq4sO4E4AHQA8ph/nSG1kGIZpSS4Y2xPjytujr6K5czJkhO6HXZJpnzmloAshNgAYqdj+hOn1DAAz/B0awzCMfxBR2mIOqKtANhaZMZOpoJ+rLTIMwzQTf7hwJC6f0AfHlLfPyPl56T/DMEwz0aOsEHedOyxj5+cInWEYppXAgs4wDNNKYEFnGIZJgkxbbO6VrY2BPXSGYZgk9GxXiJ+dejTOG92jpYeSEhZ0hmGYJBARbjxlQEsPwxNsuTAMw7QSPAk6EVUQ0QoiWkZEixX7iYgeIaL1RPQVEY3xf6gMwzBMMvxqcDENwAD9azyAx8ENLhiGYZoVvyyXcwH8XWgsAFBGRN18OjfDMAzjAb8aXPQAsMX0fqu+jWEYhmkmfG1wkQr9ZnANAPTu3TvdjzMMwzBJ8BShmxtcAJANLsxsA9DL9L6nvs1+nieFEOOEEOM6derUuBEzDMMwSnxpcAHgNQCX69kuxwE4IITY4ftoGYZhGFdIpKjaTkT9oEXlQKLBxb3mBhekdbX4E4AzANQCuEoI4UhvtJ13D4BNjRx3RwBuGTfZAo+x6WT7+AAeox9k+/iA7BpjHyGE0uJIKejZCBEtzvaOSDzGppPt4wN4jH6Q7eMDcmOMAK8UZRiGaTWwoDMMw7QSclXQn2zpAXiAx9h0sn18AI/RD7J9fEBujDE3PXSGYRjGSa5G6AzDMIwNFnSGYZhWQs4JOhGdQURr9VK9M1twHH8hot1E9LVpW3sieo+I1un/ttO3N3t5YSLqRUTziGgVEa0kop9m4RgLiGgRES3Xx/gbfXtfIlqoj+UFIsrTt+fr79fr+8szPUb9ukEiWkpEb2Tp+BzlrbPp96xft4yIXiSiNUS0mogmZMsYiWig/rOTXweJ6KZsGV9aCCFy5gtAEMC3APoByAOwHMCQFhrLiQDGAPjatO33AGbqr2cCuE9/fSaAtwEQgOMALGyG8XUDMEZ/3RbANwCGZNkYCUCx/joMYKF+7f8AuFjf/gSA6/TX1wN4Qn99MYAXmul3fTOAfwF4Q3+fbeOrANDRti1rfs/6df8GYIb+Og9AWbaNUb92EMBOAH2ycXwpx9/SA0jzhz0BwDum978A8IsWHE+5TdDXAuimv+4GYK3++s8ALlEd14xjfRXAqdk6RgBtAHwJrY5+JYCQ/XcO4B0AE/TXIf04yvC4egKYC+BkAG/of8RZMz79WipBz5rfM4BSABvtP4tsGqPpWqcB+DRbx5fqK9csl2wv09tFJGrY7ATQRX/douPWH/1HQ4uAs2qMup2xDMBuAO9BewLbL4SIKsZhjFHffwBAhwwP8SEAPwcQ1993yLLxAery1tn0e+4LYA+AZ3Xr6mnS6kJl0xglFwN4Xn+djeNLSq4Jes4gtFt3i+eEElExgJcA3CSEOGjelw1jFELEhBCjoEXCxwIY1JLjMUNEZwHYLYRY0tJjScEkIcQYaJ3DbiCiE807s+D3HIJmTz4uhBgNoAaahWGQBWOEPhdyDoD/2vdlw/i8kGuC7qlMbwuyi/ROTfq/u/XtLTJuIgpDE/N/CiFezsYxSoQQ+wHMg2ZhlBGRrNVvHocxRn1/KYC9GRzWRADnEFEFgH9Ds10ezqLxAXAtb51Nv+etALYKIRbq71+EJvDZNEZAuyF+KYTYpb/PtvGlJNcE/QsAA/Qsgzxoj0evtfCYzLwG4Ar99RXQfGu5vVnLCxMRAXgGwGohxANZOsZORFSmvy6E5vGvhibsF7iMUY79AgAf6JFTRhBC/EII0VMIUQ7t/9oHQogfZMv4gKTlrbPm9yyE2AlgCxEN1DedAmBVNo1R5xIk7BY5jmwaX2pa2sRvxKTFmdAyNr4F8MsWHMfzAHYAiECLQK6G5pfOBbAOwPsA2uvHEoBH9TGvADCuGcY3Cdoj4lcAlulfZ2bZGEcAWKqP8WsAd+rb+wFYBGA9tMfffH17gf5+vb6/XzP+vk9CIssla8anj2W5/rVS/k1k0+9Zv+4oAIv13/UrANpl0xgBFEF7mio1bcua8Xn94qX/DMMwrYRcs1wYhmEYF1jQGYZhWgks6AzDMK0EFnSGYZhWAgs6wzBMK4EFnWEYppXAgs4wDNNK+P+icBP5DOF1OQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluateRandomly(encoder1, attn_decoder1)"
      ],
      "metadata": {
        "id": "5XD28gpOeIY9",
        "outputId": "f1b67685-3b55-497c-e7d3-493a80dc1b3f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ">  they do not correspond with our potential .\n",
            "=  они не совпадают с нашими возможностями .\n",
            "<  не не не не . . . . <EOS>\n",
            "\n",
            "> again the next day after john stood and two of his disciples \n",
            "= на следующии день иоанн снова был с двумя учениками .\n",
            "< и после этого и и и и и   <EOS>\n",
            "\n",
            "> the quiver rattleth upon him the glittering spear and the javelin .\n",
            "= скрежещет на него колчан сверкает копье и пика .\n",
            "< на него на на на и и . . . <EOS>\n",
            "\n",
            "> purpose opens the garage door of shepherd s house .\n",
            "= использование открывает двери гаража дома шепердов .\n",
            "< дом дом дом дом . . . <EOS>\n",
            "\n",
            "> but they went out and gave news of him in all that land .\n",
            "= а они выидя разгласили о нем по всеи земле тои .\n",
            "< они они они они все они во во . . <EOS>\n",
            "\n",
            "> sunbathing lawn garden meadow football lounge chairs free of charge\n",
            "= лужаика для загара сад луг футбол бесплатные шезлонги\n",
            "<  музеи музеи и <EOS>\n",
            "\n",
            "> in spain we do not take advantage of our potentialities .\n",
            "= в испании мы не используем наши возможности .\n",
            "< мы не мы мы не мы не . . <EOS>\n",
            "\n",
            "> it is located in a historical building opposite the . . .\n",
            "= она находится в историческом здании на бульварном к . . .\n",
            "< в это в в . . <EOS>\n",
            "\n",
            "> therefore is justice far from us neither doth righteousness overtake us \n",
            "= потому то и далек от нас суд и правосудие не достигает до нас \n",
            "< от нас из из от не . . . <EOS>\n",
            "\n",
            "> and when they are made low and crushed by trouble and sorrow \n",
            "= от угнетении бедствии и скорби .\n",
            "<  они и и и  <EOS>\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(encoder1.state_dict(), \"./\")\n",
        "torch.save(attn_decoder1.state_dict(), \"./\")"
      ],
      "metadata": {
        "id": "5sIaTmQzuRL_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Dl2kG3sfaCm"
      },
      "source": [
        "Seq2Seq Network using Transformer\n",
        "---------------------------------\n",
        "\n",
        "Transformer is a Seq2Seq model introduced in `“Attention is all you\n",
        "need” <https://papers.nips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf>`__\n",
        "paper for solving machine translation tasks. \n",
        "Below, we will create a Seq2Seq network that uses Transformer. The network\n",
        "consists of three parts. First part is the embedding layer. This layer converts tensor of input indices\n",
        "into corresponding tensor of input embeddings. These embedding are further augmented with positional\n",
        "encodings to provide position information of input tokens to the model. The second part is the \n",
        "actual `Transformer <https://pytorch.org/docs/stable/generated/torch.nn.Transformer.html>`__ model. \n",
        "Finally, the output of Transformer model is passed through linear layer\n",
        "that give un-normalized probabilities for each token in the target language. \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zmVDYNWsfaCp"
      },
      "outputs": [],
      "source": [
        "from torch import Tensor\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.nn import Transformer\n",
        "import math\n",
        "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "# helper Module that adds positional encoding to the token embedding to introduce a notion of word order.\n",
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self,\n",
        "                 emb_size: int,\n",
        "                 dropout: float,\n",
        "                 maxlen: int = 5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        den = torch.exp(- torch.arange(0, emb_size, 2)* math.log(10000) / emb_size)\n",
        "        pos = torch.arange(0, maxlen).reshape(maxlen, 1)\n",
        "        pos_embedding = torch.zeros((maxlen, emb_size))\n",
        "        pos_embedding[:, 0::2] = torch.sin(pos * den)\n",
        "        pos_embedding[:, 1::2] = torch.cos(pos * den)\n",
        "        pos_embedding = pos_embedding.unsqueeze(-2)\n",
        "\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.register_buffer('pos_embedding', pos_embedding)\n",
        "\n",
        "    def forward(self, token_embedding: Tensor):\n",
        "        return self.dropout(token_embedding + self.pos_embedding[:token_embedding.size(0), :])\n",
        "\n",
        "# helper Module to convert tensor of input indices into corresponding tensor of token embeddings\n",
        "class TokenEmbedding(nn.Module):\n",
        "    def __init__(self, vocab_size: int, emb_size):\n",
        "        super(TokenEmbedding, self).__init__()\n",
        "        self.embedding = nn.Embedding(vocab_size, emb_size)\n",
        "        self.emb_size = emb_size\n",
        "\n",
        "    def forward(self, tokens: Tensor):\n",
        "        return self.embedding(tokens.long()) * math.sqrt(self.emb_size)\n",
        "\n",
        "# Seq2Seq Network \n",
        "class Seq2SeqTransformer(nn.Module):\n",
        "    def __init__(self,\n",
        "                 num_encoder_layers: int,\n",
        "                 num_decoder_layers: int,\n",
        "                 emb_size: int,\n",
        "                 nhead: int,\n",
        "                 src_vocab_size: int,\n",
        "                 tgt_vocab_size: int,\n",
        "                 dim_feedforward: int = 512,\n",
        "                 dropout: float = 0.1):\n",
        "        super(Seq2SeqTransformer, self).__init__()\n",
        "        self.transformer = Transformer(d_model=emb_size,\n",
        "                                       nhead=nhead,\n",
        "                                       num_encoder_layers=num_encoder_layers,\n",
        "                                       num_decoder_layers=num_decoder_layers,\n",
        "                                       dim_feedforward=dim_feedforward,\n",
        "                                       dropout=dropout)\n",
        "        self.generator = nn.Linear(emb_size, tgt_vocab_size)\n",
        "        self.src_tok_emb = TokenEmbedding(src_vocab_size, emb_size)\n",
        "        self.tgt_tok_emb = TokenEmbedding(tgt_vocab_size, emb_size)\n",
        "        self.positional_encoding = PositionalEncoding(\n",
        "            emb_size, dropout=dropout)\n",
        "\n",
        "    def forward(self,\n",
        "                src: Tensor,\n",
        "                trg: Tensor,\n",
        "                src_mask: Tensor,\n",
        "                tgt_mask: Tensor,\n",
        "                src_padding_mask: Tensor,\n",
        "                tgt_padding_mask: Tensor,\n",
        "                memory_key_padding_mask: Tensor):\n",
        "        src_emb = self.positional_encoding(self.src_tok_emb(src))\n",
        "        tgt_emb = self.positional_encoding(self.tgt_tok_emb(trg))\n",
        "        outs = self.transformer(src_emb, tgt_emb, src_mask, tgt_mask, None, \n",
        "                                src_padding_mask, tgt_padding_mask, memory_key_padding_mask)\n",
        "        return self.generator(outs)\n",
        "\n",
        "    def encode(self, src: Tensor, src_mask: Tensor):\n",
        "        return self.transformer.encoder(self.positional_encoding(\n",
        "                            self.src_tok_emb(src)), src_mask)\n",
        "\n",
        "    def decode(self, tgt: Tensor, memory: Tensor, tgt_mask: Tensor):\n",
        "        return self.transformer.decoder(self.positional_encoding(\n",
        "                          self.tgt_tok_emb(tgt)), memory,\n",
        "                          tgt_mask)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LA03OmFAfaCs"
      },
      "source": [
        "During training, we need a subsequent word mask that will prevent model to look into\n",
        "the future words when making predictions. We will also need masks to hide\n",
        "source and target padding tokens. Below, let's define a function that will take care of both. \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AJ3cZevUfaCt"
      },
      "outputs": [],
      "source": [
        "def generate_square_subsequent_mask(sz):\n",
        "    mask = (torch.triu(torch.ones((sz, sz), device=DEVICE)) == 1).transpose(0, 1)\n",
        "    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "    return mask\n",
        "\n",
        "\n",
        "def create_mask(src, tgt):\n",
        "    src_seq_len = src.shape[0]\n",
        "    tgt_seq_len = tgt.shape[0]\n",
        "\n",
        "    tgt_mask = generate_square_subsequent_mask(tgt_seq_len)\n",
        "    src_mask = torch.zeros((src_seq_len, src_seq_len),device=DEVICE).type(torch.bool)\n",
        "\n",
        "    src_padding_mask = (src == PAD_IDX).transpose(0, 1)\n",
        "    tgt_padding_mask = (tgt == PAD_IDX).transpose(0, 1)\n",
        "    return src_mask, tgt_mask, src_padding_mask, tgt_padding_mask"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6jsbCwW5faCw"
      },
      "source": [
        "Let's now define the parameters of our model and instantiate the same. Below, we also \n",
        "define our loss function which is the cross-entropy loss and the optmizer used for training.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Zi2UODBFfaCy"
      },
      "outputs": [],
      "source": [
        "from torch.optim.lr_scheduler import ExponentialLR\n",
        "torch.manual_seed(0)\n",
        "\n",
        "SRC_VOCAB_SIZE = len(vocab_transform[SRC_LANGUAGE])\n",
        "TGT_VOCAB_SIZE = len(vocab_transform[TGT_LANGUAGE])\n",
        "EMB_SIZE = 512\n",
        "NHEAD = 8\n",
        "FFN_HID_DIM = 512\n",
        "BATCH_SIZE = 64 \n",
        "NUM_ENCODER_LAYERS = 3\n",
        "NUM_DECODER_LAYERS = 3\n",
        "\n",
        "transformer = Seq2SeqTransformer(NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, EMB_SIZE, \n",
        "                                 NHEAD, SRC_VOCAB_SIZE, TGT_VOCAB_SIZE, FFN_HID_DIM)\n",
        "\n",
        "for p in transformer.parameters():\n",
        "    if p.dim() > 1:\n",
        "        nn.init.xavier_uniform_(p)\n",
        "\n",
        "transformer = transformer.to(DEVICE)\n",
        "\n",
        "loss_fn = torch.nn.CrossEntropyLoss(ignore_index=PAD_IDX)\n",
        "\n",
        "optimizer = torch.optim.Adam(transformer.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-7)\n",
        "scheduler = ExponentialLR(optimizer, gamma=0.99) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F_kIa1tkfaCz"
      },
      "source": [
        "Collation\n",
        "---------\n",
        "\n",
        "As seen in the ``Data Sourcing and Processing`` section, our data iterator yields a pair of raw strings. \n",
        "We need to convert these string pairs into the batched tensors that can be processed by our ``Seq2Seq`` network \n",
        "defined previously. Below we define our collate function that convert batch of raw strings into batch tensors that\n",
        "can be fed directly into our model.   \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cwNDu-befaC0"
      },
      "outputs": [],
      "source": [
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "# helper function to club together sequential operations\n",
        "def sequential_transforms(*transforms):\n",
        "    def func(txt_input):\n",
        "        for transform in transforms:\n",
        "            txt_input = transform(txt_input)\n",
        "        return txt_input\n",
        "    return func\n",
        "\n",
        "# function to add BOS/EOS and create tensor for input sequence indices\n",
        "def tensor_transform(token_ids: List[int]):\n",
        "    return torch.cat((torch.tensor([BOS_IDX]), \n",
        "                      torch.tensor(token_ids), \n",
        "                      torch.tensor([EOS_IDX])))\n",
        "\n",
        "# src and tgt language text transforms to convert raw strings into tensors indices\n",
        "text_transform = {}\n",
        "for ln in [SRC_LANGUAGE, TGT_LANGUAGE]:\n",
        "    text_transform[ln] = sequential_transforms(token_transform[ln], #Tokenization\n",
        "                                               vocab_transform[ln], #Numericalization\n",
        "                                               tensor_transform) # Add BOS/EOS and create tensor\n",
        "\n",
        "\n",
        "# function to collate data samples into batch tesors\n",
        "def collate_fn(batch):\n",
        "    src_batch, tgt_batch = [], []\n",
        "    for src_sample, tgt_sample in batch:\n",
        "        src_batch.append(text_transform[SRC_LANGUAGE](src_sample.rstrip(\"\\n\")))\n",
        "        tgt_batch.append(text_transform[TGT_LANGUAGE](tgt_sample.rstrip(\"\\n\")))\n",
        "\n",
        "    src_batch = pad_sequence(src_batch, padding_value=PAD_IDX)\n",
        "    tgt_batch = pad_sequence(tgt_batch, padding_value=PAD_IDX)\n",
        "    return src_batch, tgt_batch"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "69SaTchafaC2"
      },
      "source": [
        "Let's define training and evaluation loop that will be called for each \n",
        "epoch.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Define iterators"
      ],
      "metadata": {
        "id": "QpSBiO7245G5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "N_train = int(N * 0.9)\n",
        "\n",
        "complete_ds = list(zip(lines_eng, lines_rus))\n",
        "train_iter = complete_ds[:N_train]\n",
        "test_iter = complete_ds[N_train:] \n",
        "N_train"
      ],
      "metadata": {
        "id": "ga1v85o647Kc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ae59357-14a9-4842-904d-720e2f390e3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "90000"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6_A7qllEfaC3"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader\n",
        "from tqdm import tqdm\n",
        "\n",
        "def train_epoch(model, optimizer):\n",
        "    model.train()\n",
        "    losses = 0\n",
        "    train_dataloader = DataLoader(train_iter, batch_size=BATCH_SIZE, collate_fn=collate_fn)\n",
        "\n",
        "    with tqdm(train_dataloader) as t:\n",
        "        for src, tgt in t:\n",
        "            src = src.to(DEVICE)\n",
        "            tgt = tgt.to(DEVICE)\n",
        "\n",
        "            tgt_input = tgt[:-1, :]\n",
        "\n",
        "            src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input)\n",
        "\n",
        "            logits = model(src, tgt_input, src_mask, tgt_mask,src_padding_mask, tgt_padding_mask, src_padding_mask)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            tgt_out = tgt[1:, :]\n",
        "            loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1))\n",
        "            loss.backward()\n",
        "\n",
        "            optimizer.step()\n",
        "            losses += loss.item()\n",
        "            t.set_description(f'Loss: {loss.item()}')\n",
        "\n",
        "    return losses / len(train_dataloader)\n",
        "\n",
        "\n",
        "def evaluate(model):\n",
        "    model.eval()\n",
        "    losses = 0\n",
        "\n",
        "    val_dataloader = DataLoader(test_iter, batch_size=BATCH_SIZE,\n",
        "                                collate_fn=collate_fn)\n",
        "\n",
        "    for src, tgt in val_dataloader:\n",
        "        src = src.to(DEVICE)\n",
        "        tgt = tgt.to(DEVICE)\n",
        "\n",
        "        tgt_input = tgt[:-1, :]\n",
        "\n",
        "        src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input)\n",
        "\n",
        "        logits = model(src, tgt_input, src_mask, tgt_mask,src_padding_mask, tgt_padding_mask, src_padding_mask)\n",
        "        \n",
        "        tgt_out = tgt[1:, :]\n",
        "        loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1))\n",
        "        losses += loss.item()\n",
        "\n",
        "    return losses / len(val_dataloader)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kjpylHscfaC4"
      },
      "source": [
        "Now we have all the ingredients to train our model. Let's do it!\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OS_M1IcFfaC6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "97b6e5be-9a88-4640-ac4c-d4310c5f92a2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 7.437880039215088: 100%|██████████| 1407/1407 [08:28<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1, Train loss: 6.908, Val loss: 7.172, Epoch time = 508.925s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 6.653111934661865: 100%|██████████| 1407/1407 [08:27<00:00,  2.77it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2, Train loss: 5.880, Val loss: 6.934, Epoch time = 507.990s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 6.099935531616211: 100%|██████████| 1407/1407 [08:27<00:00,  2.77it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 3, Train loss: 5.364, Val loss: 6.864, Epoch time = 507.735s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 5.528937816619873: 100%|██████████| 1407/1407 [08:28<00:00,  2.77it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 4, Train loss: 4.972, Val loss: 6.838, Epoch time = 508.075s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 5.236104488372803: 100%|██████████| 1407/1407 [08:30<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 5, Train loss: 4.657, Val loss: 6.766, Epoch time = 510.297s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 4.919432640075684: 100%|██████████| 1407/1407 [08:30<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 6, Train loss: 4.393, Val loss: 6.679, Epoch time = 510.561s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 4.658354759216309: 100%|██████████| 1407/1407 [08:30<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 7, Train loss: 4.169, Val loss: 6.659, Epoch time = 510.627s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 4.420555114746094: 100%|██████████| 1407/1407 [08:30<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 8, Train loss: 3.977, Val loss: 6.642, Epoch time = 510.699s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 4.295312404632568: 100%|██████████| 1407/1407 [08:30<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 9, Train loss: 3.808, Val loss: 6.638, Epoch time = 510.903s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 4.038195610046387: 100%|██████████| 1407/1407 [08:30<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 10, Train loss: 3.657, Val loss: 6.679, Epoch time = 510.753s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.890901803970337: 100%|██████████| 1407/1407 [08:30<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 11, Train loss: 3.523, Val loss: 6.773, Epoch time = 510.696s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.823850393295288: 100%|██████████| 1407/1407 [08:30<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 12, Train loss: 3.404, Val loss: 6.846, Epoch time = 510.884s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.6336758136749268: 100%|██████████| 1407/1407 [08:31<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 13, Train loss: 3.291, Val loss: 7.028, Epoch time = 511.612s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.481980800628662: 100%|██████████| 1407/1407 [08:31<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 14, Train loss: 3.190, Val loss: 7.228, Epoch time = 511.956s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.4601991176605225: 100%|██████████| 1407/1407 [08:32<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 15, Train loss: 3.100, Val loss: 7.111, Epoch time = 512.285s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.3359272480010986: 100%|██████████| 1407/1407 [08:31<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 16, Train loss: 3.012, Val loss: 6.949, Epoch time = 511.764s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.338839054107666: 100%|██████████| 1407/1407 [08:31<00:00,  2.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 17, Train loss: 2.925, Val loss: 6.891, Epoch time = 511.369s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss: 3.165442705154419: 100%|██████████| 1407/1407 [08:29<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 18, Train loss: 2.847, Val loss: 6.968, Epoch time = 509.465s\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from timeit import default_timer as timer\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "NUM_EPOCHS = 18\n",
        "\n",
        "sw = SummaryWriter()\n",
        "for epoch in range(1, NUM_EPOCHS+1):\n",
        "    start_time = timer()\n",
        "    train_loss = train_epoch(transformer, optimizer)\n",
        "    end_time = timer()\n",
        "    val_loss = evaluate(transformer)\n",
        "    print((f\"Epoch: {epoch}, Train loss: {train_loss:.3f}, Val loss: {val_loss:.3f}, \"f\"Epoch time = {(end_time - start_time):.3f}s\"))\n",
        "\n",
        "    sw.add_scalar('Loss/train', train_loss)\n",
        "    sw.add_scalar('Loss/val', val_loss)\n",
        "    scheduler.step()\n",
        "\n",
        "# Save last ckpt\n",
        "PATH = 'transformer.ckpt'\n",
        "torch.save(transformer.state_dict(), PATH)\n",
        "\n",
        "\n",
        "\n",
        "# function to generate output sequence using greedy algorithm \n",
        "def greedy_decode(model, src, src_mask, max_len, start_symbol):\n",
        "    src = src.to(DEVICE)\n",
        "    src_mask = src_mask.to(DEVICE)\n",
        "\n",
        "    memory = model.encode(src, src_mask)\n",
        "    ys = torch.ones(1, 1).fill_(start_symbol).type(torch.long).to(DEVICE)\n",
        "    for i in range(max_len-1):\n",
        "        memory = memory.to(DEVICE)\n",
        "        tgt_mask = (generate_square_subsequent_mask(ys.size(0))\n",
        "                    .type(torch.bool)).to(DEVICE)\n",
        "        out = model.decode(ys, memory, tgt_mask)\n",
        "        out = out.transpose(0, 1)\n",
        "        prob = model.generator(out[:, -1])\n",
        "        _, next_word = torch.max(prob, dim=1)\n",
        "        next_word = next_word.item()\n",
        "\n",
        "        ys = torch.cat([ys,\n",
        "                        torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=0)\n",
        "        if next_word == EOS_IDX:\n",
        "            break\n",
        "    return ys\n",
        "\n",
        "\n",
        "# actual function to translate input sentence into target language\n",
        "def translate(model: torch.nn.Module, src_sentence: str):\n",
        "    model.eval()\n",
        "    src = text_transform[SRC_LANGUAGE](src_sentence).view(-1, 1)\n",
        "    num_tokens = src.shape[0]\n",
        "    src_mask = (torch.zeros(num_tokens, num_tokens)).type(torch.bool)\n",
        "    tgt_tokens = greedy_decode(\n",
        "        model,  src, src_mask, max_len=num_tokens + 5, start_symbol=BOS_IDX).flatten()\n",
        "    return \" \".join(vocab_transform[TGT_LANGUAGE].lookup_tokens(list(tgt_tokens.cpu().numpy()))).replace(\"<bos>\", \"\").replace(\"<eos>\", \"\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mlSa0OcFfaC7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3255c0b5-66ae-45c5-b510-e520de994360"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Я люблю вас \n"
          ]
        }
      ],
      "source": [
        "print(translate(transformer, \"I love you\"))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, \"This is success\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q64guJC6dbF1",
        "outputId": "323570eb-1505-4088-fdab-454925ac299a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Это успех , это успех \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'Where is a bathroom'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yf1jjIaLduAc",
        "outputId": "b7addeea-7db8-4349-e4ca-ca6f9108baee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Где есть ванная комната \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'Sad teddy bear'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mnGHGmT2d6Fw",
        "outputId": "689efa24-7ddc-433b-fd6c-74d82b0b20a8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " ЦИК подал отчет в отношении \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'I hate you'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rHIZUIqieSg6",
        "outputId": "3790b744-066d-4256-b4fd-c985f958f1ac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Я не люблю вас \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'How can I pay for this?'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lgt8USP_eh_z",
        "outputId": "cb890dce-2fec-4d41-b3e7-c75ec632d968"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Как я могу платить за это ? \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'Do you want to play a game with me?'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nJONJxk1e2fD",
        "outputId": "b384b789-74c2-4504-fc0c-b054b9a7fdbc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Вы хотите играть в игре с меня ? \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'How can you really?'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v5tePgXnfVSa",
        "outputId": "8fed7601-b37c-4560-e2e3-729c5ecf9bf4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Как вы можете действительно ? \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'I love Valdimir Putin'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tHoB7mgfgxTq",
        "outputId": "dc4ee56b-b365-4f72-d02f-991888bac96d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Я люблю СМИ \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'What is the strength in, brother?'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SpHHYthVh56N",
        "outputId": "5321d1a5-da29-448d-97cf-c01c37a3276a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Что такое сила , брат , с которого нет силы ? \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translate(transformer, 'As far as I know he didn\\'t take your keys'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_y1b2jcWijXN",
        "outputId": "46d2e0ef-06b2-4681-e93a-9543f3c490a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Как я не знаю , он не получил свою партия \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Kind of working !!!"
      ],
      "metadata": {
        "id": "uFpoYB57iT_2"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7u4fzKFAfaC7"
      },
      "source": [
        "References\n",
        "----------\n",
        "\n",
        "1. Attention is all you need paper.\n",
        "   https://papers.nips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf\n",
        "2. The annotated transformer. https://nlp.seas.harvard.edu/2018/04/03/attention.html#positional-encoding\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.13"
    },
    "colab": {
      "name": "translation_transformer.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "CLaeRh89Jt5w",
        "3Em0Fn6bJgrY"
      ],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}