{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d52578aa",
   "metadata": {
    "tags": [
     "hide-cell"
    ]
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42c25d64",
   "metadata": {},
   "source": [
    "# Implementing Ensemble Methods with Python\n",
    "\n",
    "In this section, we will see how to implement the ensemble methods we learned in the previous section using Python. We will work with the Titanic dataset, which contains information about passengers and whether they survived. \n",
    "\n",
    "Earlier in the chapter, we preprocessed the data by handling missing values and encoding categorical variables. We work with the same preprocessed dataset. If you click on the following code cell shows, it will show you the pre processing we performed. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "57308424",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>29.1250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>886</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13.0000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>887</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30.0000</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30.0000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.7500</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>712 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Survived  Pclass   Age  SibSp  Parch     Fare  Sex  Embarked\n",
       "0           0       3  22.0      1      0   7.2500    1         2\n",
       "1           1       1  38.0      1      0  71.2833    0         0\n",
       "2           1       3  26.0      0      0   7.9250    0         2\n",
       "3           1       1  35.0      1      0  53.1000    0         2\n",
       "4           0       3  35.0      0      0   8.0500    1         2\n",
       "..        ...     ...   ...    ...    ...      ...  ...       ...\n",
       "885         0       3  39.0      0      5  29.1250    0         1\n",
       "886         0       2  27.0      0      0  13.0000    1         2\n",
       "887         1       1  19.0      0      0  30.0000    0         2\n",
       "889         1       1  26.0      0      0  30.0000    1         0\n",
       "890         0       3  32.0      0      0   7.7500    1         1\n",
       "\n",
       "[712 rows x 8 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing the libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Load the dataset\n",
    "Titanic_df = pd.read_csv(\"Titanic-Dataset.csv\")\n",
    "Titanic_df\n",
    "\n",
    "# Drop 'Cabin' column because most of its values were missing\n",
    "Titanic_df = Titanic_df.drop('Cabin', axis=1)\n",
    "\n",
    "# Drop rows with missing values because `Age` and `Embarked` had a few missing values\n",
    "Titanic_df = Titanic_df.dropna()\n",
    "\n",
    "# Removing the Identifier Columns\n",
    "Titanic_df = Titanic_df.drop(['PassengerId', 'Name', 'Ticket'], axis=1)\n",
    "\n",
    "# Convert the string labels to numerical labels\n",
    "Titanic_df['Sex_coded']=pd.Categorical(Titanic_df['Sex']).codes\n",
    "Titanic_df['Embarked_coded']=pd.Categorical(Titanic_df['Embarked']).codes\n",
    "\n",
    "# Drop 'Sex' and 'Embarked' columns \n",
    "Titanic_df = Titanic_df.drop(['Sex', 'Embarked'], axis=1)\n",
    "\n",
    "# Rename the columns 'Sex_coded' and 'Embarked_coded' \n",
    "Titanic_df = Titanic_df.rename(columns={'Sex_coded': 'Sex', 'Embarked_coded': 'Embarked'})\n",
    "\n",
    "Titanic_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0af71204",
   "metadata": {},
   "source": [
    "Before building our ensemble models, we need to split the data into training and test sets. We make this split using the `train_test_split` function from scikit-learn's `model_selection` module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa20637f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = Titanic_df.drop(columns=['Survived'])\n",
    "y = Titanic_df['Survived']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec77ad0c",
   "metadata": {},
   "source": [
    "The function takes our feature matrix `X` and target variable `y`, and returns four datasets: `X_train` and `y_train` for training, and `X_test` and `y_test` for testing. The `test_size=0.2` parameter allocates 20% of the data for testing. The `random_state=10` parameter ensures reproducibility by producing the same random split each time.\n",
    "\n",
    "We will implement three ensemble methods: Bagging, Random Forests, and AdaBoost. We will train each method on the training data, evaluate their performance on the test data, and compare their results to see which performs best for predicting Titanic survival."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fc26e64",
   "metadata": {},
   "source": [
    "## Bagging\n",
    "\n",
    "We can implement bagging in Python using scikit-learn's `BaggingClassifier`. See [here](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.BaggingClassifier.html) for documentation.\n",
    "\n",
    "The parameters we use in creating our bagging model are:\n",
    "- **`estimator`**: The base model to use. Here, we use `DecisionTreeClassifier()` i.e. our building blocks are decision trees.\n",
    "- **`n_estimators`**: The number of bootstrap training datasets and trees to create. We use 100 trees.\n",
    "- **`random_state`**: Ensures we get the same results every time we run the code, making it reproducible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "77e86b4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "# Create a bagging classifier with decision trees as weak learners\n",
    "bagging_model = BaggingClassifier(\n",
    "    estimator=DecisionTreeClassifier(),\n",
    "    n_estimators=100,\n",
    "    random_state=42\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b172739c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BaggingClassifier(estimator=DecisionTreeClassifier(), n_estimators=100,\n",
       "                  random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;BaggingClassifier<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.ensemble.BaggingClassifier.html\">?<span>Documentation for BaggingClassifier</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>BaggingClassifier(estimator=DecisionTreeClassifier(), n_estimators=100,\n",
       "                  random_state=42)</pre></div> </div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">estimator: DecisionTreeClassifier</label><div class=\"sk-toggleable__content fitted\"><pre>DecisionTreeClassifier()</pre></div> </div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;DecisionTreeClassifier<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.tree.DecisionTreeClassifier.html\">?<span>Documentation for DecisionTreeClassifier</span></a></label><div class=\"sk-toggleable__content fitted\"><pre>DecisionTreeClassifier()</pre></div> </div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "BaggingClassifier(estimator=DecisionTreeClassifier(), n_estimators=100,\n",
       "                  random_state=42)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model\n",
    "bagging_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca62551e",
   "metadata": {},
   "source": [
    "Lets see how the 100 trees in our bagging ensemble vote for the first passenger in our test set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "07f33bc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trees voting 'Survived': 33\n",
      "Trees voting 'Not Survived': 67\n"
     ]
    }
   ],
   "source": [
    "# Get predictions from all trees for the first test example\n",
    "tree_predictions = [tree.predict([X_test.iloc[0].values]) for tree in bagging_model.estimators_]\n",
    "votes = np.array(tree_predictions).flatten()\n",
    "\n",
    "print(f\"Trees voting 'Survived': {np.sum(votes == 1)}\")\n",
    "print(f\"Trees voting 'Not Survived': {np.sum(votes == 0)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3f96e77",
   "metadata": {},
   "source": [
    "Since we use majority vote in classification, we would therefore predict the first passenger to not survive. \n",
    "We now make the predictions on the whole test set and evaluate the model's overall accuracy. For that, we import `accuracy_score` from `sklearn.metrics`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f6240a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bagging Accuracy: 0.8042\n"
     ]
    }
   ],
   "source": [
    "# Make predictions\n",
    "y_pred_bagging = bagging_model.predict(X_test)\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Evaluate accuracy\n",
    "bagging_accuracy = accuracy_score(y_test, y_pred_bagging)\n",
    "print(f\"Bagging Accuracy: {bagging_accuracy:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a9d887f",
   "metadata": {},
   "source": [
    "This means our bagging model correctly predicts survival for approximately 80% of the passengers in the test set.\n",
    "\n",
    "We can also visualize the feature importance for each of the features used in the modeling. Each tree calculates its own feature importance based on the bootstrap sample it was trained on. We average these importances across all trees to get a more robust estimate of which features are most important for prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "592cf01b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAHFCAYAAAA0SmdSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCFUlEQVR4nO3deVxWZf7/8feN4M2OiguagDtoiku4p0immFhZmplZkMvYak7awqgpWS6ZY2kuZSqOZYblmDqllYajo+ZKizotJmGTW24gKbJcvz/8cv+6BVw4IIKv5+NxP4Zz7utc53MujsO765xz3zZjjBEAAACKzKW0CwAAACjrCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUQBEkJCTIZrMV+Bo1alSJ7HPv3r0aP368UlJSSqR/K1JSUmSz2fTaa6+VdilFtnnzZo0fP16nTp0q7VKumdjYWNWpU6fY+ss7D/788vX1VfPmzfX6668rJyen2PZl1fjx42Wz2Uq7DJQjrqVdAFCWLVy4UKGhoU7ratWqVSL72rt3r+Lj49WlS5di/SOICzZv3qz4+HjFxsaqUqVKpV3ONTF27Fg9/fTTxd7vU089pQEDBkiSTp06pZUrV+qvf/2rDh48qGnTphX7/opiyJAh6tGjR2mXgXKEQAVY0LRpU4WHh5d2GZZkZWXJZrPJ1fXG/L+Ds2fPyt3dvbTLKBX169cvkX6DgoLUrl07x3KPHj303Xff6f33379uAlXt2rVVu3bt0i4D5QiX/IAS9MEHH6h9+/by8vKSt7e3oqKitHv3bqc2O3bsUP/+/VWnTh15eHioTp06euCBB/TLL7842iQkJOi+++6TJEVGRjoupyQkJEiS6tSpo9jY2Hz779Kli7p06eJYTkpKks1m0+LFizVy5EjddNNNstvt+umnnyRJX3zxhbp27SpfX195enqqY8eOWrduXZGOPe+y6Pr16zV06FD5+/vL19dXDz/8sDIyMnT48GH169dPlSpVUs2aNTVq1ChlZWU5ts+7fPTqq6/qlVdeUVBQkNzd3RUeHl5gTZs2bVLXrl3l4+MjT09PdejQQf/6178KrOmzzz7ToEGDVK1aNXl6eiouLk7PPvusJKlu3bqO8U1KSpJ04ffYvXt31axZUx4eHmrcuLFeeOEFZWRkOPUfGxsrb29v/fTTT+rZs6e8vb0VGBiokSNHKjMz06ltZmamXnrpJTVu3Fju7u7y9/dXZGSkNm/e7GhjjNHs2bPVokULeXh4qHLlyurbt69+/vlnp752796tXr16qXr16rLb7apVq5aio6P166+/XvJ3VNAlP5vNpieffFKLFy9W48aN5enpqebNm2v16tWX7Oty/Pz85Obm5rTuSsdVkubNm6dGjRrJbrerSZMmWrJkSYH1//rrr+rbt698fHxUqVIlPfjgg9q+fbvTvxep4Et+derUUa9evbRmzRq1atVKHh4eCg0N1YIFC/LVs2nTJrVv317u7u666aabNHbsWL3zzjuy2WzX5WV5lDwCFWBBTk6OsrOznV55Jk6cqAceeEBNmjRRYmKiFi9erPT0dHXq1El79+51tEtJSVFISIhef/11rV27VlOmTNGhQ4fUunVr/f7775Kk6OhoTZw4UZI0a9YsbdmyRVu2bFF0dHSR6o6Li1Nqaqrmzp2rVatWqXr16nr33XfVvXt3+fr6atGiRUpMTFSVKlUUFRVV5FAlXbi04ufnp6VLl2rMmDFasmSJhg4dqujoaDVv3lwffvihYmJiNG3aNM2cOTPf9m+++abWrFmj119/Xe+++65cXFx0xx13aMuWLY42GzZs0G233abTp09r/vz5ev/99+Xj46M777xTH3zwQb4+Bw0aJDc3Ny1evFgffvihHnvsMT311FOSpOXLlzvGt1WrVpKkH3/8UT179tT8+fO1Zs0ajRgxQomJibrzzjvz9Z2VlaW77rpLXbt21ccff6xBgwZp+vTpmjJliqNNdna27rjjDk2YMEG9evXSP//5TyUkJKhDhw5KTU11tBs2bJhGjBih22+/XStWrNDs2bO1Z88edejQQUeOHJEkZWRkqFu3bjpy5IhmzZqlzz//XK+//rqCgoKUnp5epN/Zv/71L7355pt66aWX9NFHH6lKlSq655578gW5wuTm5jr+PRw/flwLFizQmjVr9NBDDzm1u9Jxffvtt/WXv/xFYWFhWr58ucaMGaP4+HhH4M2TkZGhyMhIffnll5oyZYoSExNVo0YN3X///Vd87F9//bVGjhypv/71r/r4448VFhamwYMH69///rejzTfffKNu3brpjz/+0KJFizR37lzt2rVLr7zyyhXvB+WQAXDVFi5caCQV+MrKyjKpqanG1dXVPPXUU07bpaenm4CAANOvX79C+87OzjZnzpwxXl5e5o033nCsX7ZsmZFkvvzyy3zbBAcHm5iYmHzrIyIiTEREhGP5yy+/NJJM586dndplZGSYKlWqmDvvvNNpfU5OjmnevLlp06bNJUbDmAMHDhhJZurUqY51eWN08Rj07t3bSDJ///vfnda3aNHCtGrVKl+ftWrVMmfPnnWsT0tLM1WqVDG33367Y127du1M9erVTXp6umNddna2adq0qaldu7bJzc11qunhhx/OdwxTp041ksyBAwcueay5ubkmKyvLbNiwwUgyX3/9teO9mJgYI8kkJiY6bdOzZ08TEhLiWP7HP/5hJJl58+YVup8tW7YYSWbatGlO6w8ePGg8PDzMc889Z4wxZseOHUaSWbFixSXrLkhMTIwJDg52WifJ1KhRw6SlpTnWHT582Li4uJhJkyZdsr+831lBr9jYWJOdnV3otoWNa05OjgkICDBt27Z1av/LL78YNzc3p/pnzZplJJlPP/3Uqe2wYcOMJLNw4ULHunHjxpmL/wQGBwcbd3d388svvzjWnT171lSpUsUMGzbMse6+++4zXl5e5tixY451OTk5pkmTJld0DqF8YoYKsOAf//iHtm/f7vRydXXV2rVrlZ2drYcffthp9srd3V0RERFO/2V95swZPf/882rQoIFcXV3l6uoqb29vZWRkaN++fSVSd58+fZyWN2/erBMnTigmJsap3tzcXPXo0UPbt28v8DLMlejVq5fTcuPGjSUp3+xa48aNnS5z5rn33nud7nHKm3n697//rZycHGVkZOirr75S37595e3t7WhXoUIFPfTQQ/r111/1/fffX/L4L+fnn3/WgAEDFBAQoAoVKsjNzU0RERGSlO93ZLPZ8s2whIWFOR3bp59+Knd3dw0aNKjQfa5evVo2m00DBw50+p0EBASoefPmjnOoQYMGqly5sp5//nnNnTvXafazqCIjI+Xj4+NYrlGjhqpXr17g76cgTz/9tOPfw5dffqmJEycqMTFRDzzwgFO7KxnX77//3nF5+M+CgoLUsWNHp3UbNmyQj49PvpvNL97vpbRo0UJBQUGOZXd3dzVq1Mjp2PNmRKtWrepY5+Likq9G3FhuzLtQgWLSuHHjAm9Kz7sc07p16wK3c3H5//8tM2DAAK1bt05jx45V69at5evrK5vNpp49e+rs2bMlUnfNmjULrLdv376FbnPixAl5eXld9b6qVKnitFyxYsVC1587dy7f9gEBAQWuO3/+vM6cOaP09HQZY/Idk/T/n7g8fvy40/qC2hbmzJkz6tSpk9zd3fXyyy+rUaNG8vT01MGDB3Xvvffm+x15enrmu8ndbrc7HduxY8dUq1Ytp/PgYkeOHJExRjVq1Cjw/Xr16km6cG/Shg0b9Morr+hvf/ubTp48qZo1a2ro0KEaM2ZMvvuWroS/v3++dXa7/YrPx9q1azv9u+jSpYtsNpvi4uK0du1aRUVFXfG45v3uChqHGjVq6MCBA47l48ePF9ruSl3JsRfHflD+EKiAEpD3X64ffvihgoODC213+vRprV69WuPGjdMLL7zgWJ+ZmakTJ05c8f7c3d3z3fQsSb///rvTf0Xnufhm3Lw2M2fOdHo6689K64/F4cOHC1xXsWJFeXt7y9XVVS4uLjp06FC+dr/99psk5RuDq/n8ofXr1+u3335TUlKSY/ZEkqXPq6pWrZo2bdqk3NzcQkNV1apVZbPZtHHjRtnt9nzv/3lds2bNtHTpUhlj9M033yghIUEvvfSSPDw8nM6r0hQWFibpwj1KUVFRVzyueQEnL/T/2cXnhr+/v7Zt23bZdlb5+/tfUT24sXDJDygBUVFRcnV11f79+xUeHl7gS7rwh90Yk+8P5jvvvJPvQxDz2hQ0S1CnTh198803Tut++OGHfJe6CtOxY0dVqlRJe/fuLbTevJmla2358uVOszvp6elatWqVOnXqpAoVKsjLy0tt27bV8uXLncYmNzdX7777rmrXrq1GjRpddj+FjW9e+Lr4d/TWW28V+ZjuuOMOnTt3zumps4v16tVLxhj973//K/D30axZs3zb2Gw2NW/eXNOnT1elSpW0a9euItdY3JKTkyVJ1atXl3Tl4xoSEqKAgAAlJiY6rU9NTXV6IlKSIiIilJ6erk8//dRp/dKlSy3Xf/F+1q9f73hoRLpwvi1btqxY94OyhRkqoATUqVNHL730kkaPHq2ff/5ZPXr0UOXKlXXkyBFt27ZNXl5eio+Pl6+vrzp37qypU6eqatWqqlOnjjZs2KD58+fn+3DJpk2bSrrwxJOPj4/c3d1Vt25d+fv766GHHtLAgQP1+OOPq0+fPvrll1/06quvqlq1aldUr7e3t2bOnKmYmBidOHFCffv2VfXq1XXs2DF9/fXXOnbsmObMmVPcw3RFKlSooG7duumZZ55Rbm6upkyZorS0NMXHxzvaTJo0Sd26dVNkZKRGjRqlihUravbs2Y7PPrqSGam8gPLGG28oJiZGbm5uCgkJUYcOHVS5cmU9+uijGjdunNzc3PTee+/p66+/LvIxPfDAA1q4cKEeffRRff/994qMjFRubq6++uorNW7cWP3791fHjh31l7/8RY888oh27Nihzp07y8vLS4cOHdKmTZvUrFkzPfbYY1q9erVmz56t3r17q169ejLGaPny5Tp16pS6detW5BqtSE1N1datWyVdePJuy5YtmjRpkoKDg3XvvfdK0hWPq4uLi+Lj4zVs2DD17dtXgwYN0qlTpxQfH6+aNWs6zfDFxMRo+vTpGjhwoF5++WU1aNBAn376qdauXevoqziMHj1aq1atUteuXTV69Gh5eHho7ty5jvsMi2s/KGNK8454oKzKe1ps+/btl2y3YsUKExkZaXx9fY3dbjfBwcGmb9++5osvvnC0+fXXX02fPn1M5cqVjY+Pj+nRo4f57rvvCnxy7/XXXzd169Y1FSpUcHpqKTc317z66qumXr16xt3d3YSHh5v169cX+pTfsmXLCqx3w4YNJjo62lSpUsW4ubmZm266yURHRxfaPs+lnvK7eIzynq768xNSxlx44szLyytfn1OmTDHx8fGmdu3apmLFiqZly5Zm7dq1+WrYuHGjue2224yXl5fx8PAw7dq1M6tWrXJqc7nfW1xcnKlVq5ZxcXFxeqJy8+bNpn379sbT09NUq1bNDBkyxOzatSvfk2MXH8PFx/xnZ8+eNS+++KJp2LChqVixovH39ze33Xab2bx5s1O7BQsWmLZt2zqOq379+ubhhx82O3bsMMYY89///tc88MADpn79+sbDw8P4+fmZNm3amISEhAKP8c8Ke8rviSeeyNe2sCdJ/6ygp/zc3d1No0aNzIgRI8yhQ4ec2l/puBpjzNtvv20aNGhgKlasaBo1amQWLFhg7r77btOyZUundqmpqebee+813t7exsfHx/Tp08d88sknRpL5+OOPHe0Ke8ovOjo633Fd/O/ImAvnW9u2bY3dbjcBAQHm2WefNVOmTDGSzKlTpy45TiifbMYYc20jHABcXkpKiurWraupU6eW2Pcjouw6deqUGjVqpN69e+vtt9++ZNuJEydqzJgxSk1NLdFPR+/evbtSUlL0ww8/lNg+cP3ikh8A4Lp2+PBhvfLKK4qMjJS/v79++eUXTZ8+Xenp6fm+i/DNN9+UJIWGhiorK0vr16/XjBkzNHDgwGINU88884xatmypwMBAnThxQu+9954+//xzzZ8/v9j2gbKFQAUAuK7Z7XalpKTo8ccf14kTJ+Tp6al27dpp7ty5uvnmm53aenp6avr06UpJSVFmZqaCgoL0/PPPa8yYMcVaU05Ojl588UUdPnxYNptNTZo00eLFizVw4MBi3Q/KDi75AQAAWMSjCAAAABYRqAAAACwiUAEAAFjETenXQG5urn777Tf5+Phc1VdeAACA0mOMUXp6+mW/e1MiUF0Tv/32mwIDA0u7DAAAUAQHDx687MduEKiuAR8fH0kXfiG+vr6lXA0AALgSaWlpCgwMdPwdvxQC1TWQd5nP19eXQAUAQBlzJbfrcFM6AACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALDItbQLuJE0HbdWLnbP0i4DAIByJWVydGmXwAwVAACAVQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALDohghUR48e1bBhwxQUFCS73a6AgABFRUVpy5YtpV0aAAAoB1xLu4BroU+fPsrKytKiRYtUr149HTlyROvWrdOJEydKuzQAAFAOlPsZqlOnTmnTpk2aMmWKIiMjFRwcrDZt2iguLk7R0dGSpNOnT+svf/mLqlevLl9fX9122236+uuvJUnHjh1TQECAJk6c6Ojzq6++UsWKFfXZZ5+VyjEBAIDrS7kPVN7e3vL29taKFSuUmZmZ731jjKKjo3X48GF98skn2rlzp1q1aqWuXbvqxIkTqlatmhYsWKDx48drx44dOnPmjAYOHKjHH39c3bt3L4UjAgAA1xubMcaUdhEl7aOPPtLQoUN19uxZtWrVShEREerfv7/CwsK0fv163XPPPTp69KjsdrtjmwYNGui5557TX/7yF0nSE088oS+++EKtW7fW119/re3bt8vd3b3A/WVmZjqFt7S0NAUGBipwRKJc7J4le7AAANxgUiZHl0i/aWlp8vPz0+nTp+Xr63vJtuV+hkq6cA/Vb7/9ppUrVyoqKkpJSUlq1aqVEhIStHPnTp05c0b+/v6O2Sxvb28dOHBA+/fvd/Tx2muvKTs7W4mJiXrvvfcKDVOSNGnSJPn5+TlegYGB1+IwAQBAKbkhZqgKMmTIEH3++ed6/PHHNXPmTCUlJeVrU6lSJVWtWlWStGfPHoWHhysrK0v//Oc/deeddxbaNzNUAABcO9fDDNUN8ZRfQZo0aaIVK1aoVatWOnz4sFxdXVWnTp0C254/f14PPvig7r//foWGhmrw4MH69ttvVaNGjQLb2+12p8uHAACgfCv3l/yOHz+u2267Te+++66++eYbHThwQMuWLdOrr76qu+++W7fffrvat2+v3r17a+3atUpJSdHmzZs1ZswY7dixQ5I0evRonT59WjNmzNBzzz2nxo0ba/DgwaV8ZAAA4HpR7meovL291bZtW02fPl379+9XVlaWAgMDNXToUP3tb3+TzWbTJ598otGjR2vQoEGOj0no3LmzatSooaSkJL3++uv68ssvHdN9ixcvVlhYmObMmaPHHnuslI8QAACUthv2HqprKe8aLPdQAQBQ/K6He6jK/SU/AACAkkagAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFrqVdwI3ku/ioy35bNQAAKHuYoQIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiE9Kv4aajlsrF7tnaZcBAEChUiZHl3YJZRIzVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARTdEoNq8ebMqVKigHj16lHYpAACgHLohAtWCBQv01FNPadOmTUpNTS3tcgAAQDlT7gNVRkaGEhMT9dhjj6lXr15KSEhwen/lypVq2LChPDw8FBkZqUWLFslms+nUqVOONps3b1bnzp3l4eGhwMBADR8+XBkZGdf2QAAAwHWr3AeqDz74QCEhIQoJCdHAgQO1cOFCGWMkSSkpKerbt6969+6t5ORkDRs2TKNHj3ba/ttvv1VUVJTuvfdeffPNN/rggw+0adMmPfnkk4XuMzMzU2lpaU4vAABQfpX7QDV//nwNHDhQktSjRw+dOXNG69atkyTNnTtXISEhmjp1qkJCQtS/f3/FxsY6bT916lQNGDBAI0aMUMOGDdWhQwfNmDFD//jHP3Tu3LkC9zlp0iT5+fk5XoGBgSV6jAAAoHSV60D1/fffa9u2berfv78kydXVVffff78WLFjgeL9169ZO27Rp08ZpeefOnUpISJC3t7fjFRUVpdzcXB04cKDA/cbFxen06dOO18GDB0vg6AAAwPXCtbQLKEnz589Xdna2brrpJsc6Y4zc3Nx08uRJGWNks9mctsm7HJgnNzdXw4YN0/Dhw/P1HxQUVOB+7Xa77HZ7MRwBAAAoC8ptoMrOztY//vEPTZs2Td27d3d6r0+fPnrvvfcUGhqqTz75xOm9HTt2OC23atVKe/bsUYMGDUq8ZgAAUDaV20C1evVqnTx5UoMHD5afn5/Te3379tX8+fO1fPly/f3vf9fzzz+vwYMHKzk52fEUYN7M1fPPP6927drpiSee0NChQ+Xl5aV9+/bp888/18yZM6/1YQEAgOtQub2Hav78+br99tvzhSnpwgxVcnKyTp48qQ8//FDLly9XWFiY5syZ43jKL++SXVhYmDZs2KAff/xRnTp1UsuWLTV27FjVrFnzmh4PAAC4ftnMxTcN3eBeeeUVzZ07t1hvJE9LS7vwtN+IRLnYPYutXwAAilvK5OjSLuG6kff3+/Tp0/L19b1k23J7ye9KzZ49W61bt5a/v7/+85//aOrUqZf8jCkAAICL3fCB6scff9TLL7+sEydOKCgoSCNHjlRcXFxplwUAAMqQGz5QTZ8+XdOnTy/tMgAAQBlWbm9KBwAAuFYIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAW3fBfPXMtfRcfddlvqwYAAGUPM1QAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFvFJ6ddQ03Fr5WL3LO0yAAD/J2VydGmXgHKCGSoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYFG5C1SxsbGy2Wz5Xj/99FNplwYAAMop19IuoCT06NFDCxcudFpXrVq1q+ojJydHNptNLi7lLnMCAIBiVi7Tgt1uV0BAgNPrjTfeULNmzeTl5aXAwEA9/vjjOnPmjGObhIQEVapUSatXr1aTJk1kt9v1yy+/6Pz583ruued00003ycvLS23btlVSUlLpHRwAALjulMtAVRAXFxfNmDFD3333nRYtWqT169frueeec2rzxx9/aNKkSXrnnXe0Z88eVa9eXY888oj+85//aOnSpfrmm2903333qUePHvrxxx9L6UgAAMD1plxe8lu9erW8vb0dy3fccYeWLVvmWK5bt64mTJigxx57TLNnz3asz8rK0uzZs9W8eXNJ0v79+/X+++/r119/Va1atSRJo0aN0po1a7Rw4UJNnDixwP1nZmYqMzPTsZyWllasxwcAAK4v5TJQRUZGas6cOY5lLy8vffnll5o4caL27t2rtLQ0ZWdn69y5c8rIyJCXl5ckqWLFigoLC3Nst2vXLhlj1KhRI6f+MzMz5e/vX+j+J02apPj4+GI+KgAAcL0ql4HKy8tLDRo0cCz/8ssv6tmzpx599FFNmDBBVapU0aZNmzR48GBlZWU52nl4eMhmszmWc3NzVaFCBe3cuVMVKlRw2sefZ8AuFhcXp2eeecaxnJaWpsDAwOI4NAAAcB0ql4HqYjt27FB2dramTZvmeGovMTHxstu1bNlSOTk5Onr0qDp16nTF+7Pb7bLb7UWuFwAAlC03xE3p9evXV3Z2tmbOnKmff/5Zixcv1ty5cy+7XaNGjfTggw/q4Ycf1vLly3XgwAFt375dU6ZM0SeffHINKgcAAGXBDRGoWrRoob///e+aMmWKmjZtqvfee0+TJk26om0XLlyohx9+WCNHjlRISIjuuusuffXVV1zCAwAADjZjjCntIsq7tLQ0+fn5KXBEolzsnqVdDgDg/6RMji7tEnAdy/v7ffr0afn6+l6y7Q0xQwUAAFCSCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsMi1tAu4kXwXH3XZb6sGAABlDzNUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABbxSenXUNNxa+Vi9yztMoCrljI5urRLAIDrGjNUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMCichuoYmNj1bt379IuAwAA3ACu60AVGxsrm80mm80mNzc31atXT6NGjVJGRkZplwYAAODgWtoFXE6PHj20cOFCZWVlaePGjRoyZIgyMjI0Z86c0i4NAABA0nU+QyVJdrtdAQEBCgwM1IABA/Tggw9qxYoVkqQ9e/YoOjpavr6+8vHxUadOnbR///4C+1mzZo1uvfVWVapUSf7+/urVq5dT2/Pnz+vJJ59UzZo15e7urjp16mjSpEmO98ePH6+goCDZ7XbVqlVLw4cPL9HjBgAAZcd1P0N1MQ8PD2VlZel///ufOnfurC5dumj9+vXy9fXVf/7zH2VnZxe4XUZGhp555hk1a9ZMGRkZevHFF3XPPfcoOTlZLi4umjFjhlauXKnExEQFBQXp4MGDOnjwoCTpww8/1PTp07V06VLdfPPNOnz4sL7++utCa8zMzFRmZqZjOS0trXgHAQAAXFfKVKDatm2blixZoq5du2rWrFny8/PT0qVL5ebmJklq1KhRodv26dPHaXn+/PmqXr269u7dq6ZNmyo1NVUNGzbUrbfeKpvNpuDgYEfb1NRUBQQE6Pbbb5ebm5uCgoLUpk2bQvc1adIkxcfHWzxaAABQVlz3l/xWr14tb29vubu7q3379urcubNmzpyp5ORkderUyRGmLmf//v0aMGCA6tWrJ19fX9WtW1fShbAkXbgBPjk5WSEhIRo+fLg+++wzx7b33Xefzp49q3r16mno0KH65z//WehMmCTFxcXp9OnTjlfeTBcAACifrvtAFRkZqeTkZH3//fc6d+6cli9frurVq8vDw+Oq+rnzzjt1/PhxzZs3T1999ZW++uorSRfunZKkVq1a6cCBA5owYYLOnj2rfv36qW/fvpKkwMBAff/995o1a5Y8PDz0+OOPq3PnzsrKyipwX3a7Xb6+vk4vAABQfl33gcrLy0sNGjRQcHCw02xUWFiYNm7cWGio+bPjx49r3759GjNmjLp27arGjRvr5MmT+dr5+vrq/vvv17x58/TBBx/oo48+0okTJyRduHfrrrvu0owZM5SUlKQtW7bo22+/Lb4DBQAAZVaZuofqz5588knNnDlT/fv3V1xcnPz8/LR161a1adNGISEhTm0rV64sf39/vf3226pZs6ZSU1P1wgsvOLWZPn26atasqRYtWsjFxUXLli1TQECAKlWqpISEBOXk5Kht27by9PTU4sWL5eHh4XSfFQAAuHFd9zNUhfH399f69et15swZRURE6JZbbtG8efMKvKfKxcVFS5cu1c6dO9W0aVP99a9/1dSpU53aeHt7a8qUKQoPD1fr1q2VkpKiTz75RC4uLqpUqZLmzZunjh07KiwsTOvWrdOqVavk7+9/rQ4XAABcx2zGGFPaRZR3aWlp8vPzU+CIRLnYPUu7HOCqpUyOLu0SAOCay/v7ffr06cveD11mZ6gAAACuFwQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFjkWtoF3Ei+i4+67LdVAwCAsocZKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACL+KT0a6jpuLVysXuWdhllSsrk6NIuAQCAy2KGCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsKleBymazacWKFZKklJQU2Ww2JScnl2pNAACg/CtTgero0aMaNmyYgoKCZLfbFRAQoKioKG3ZskWSdOjQId1xxx1X1edHH32ktm3bys/PTz4+Prr55ps1cuTIkigfAACUU66lXcDV6NOnj7KysrRo0SLVq1dPR44c0bp163TixAlJUkBAwFX198UXX6h///6aOHGi7rrrLtlsNu3du1fr1q0rifIBAEA5VWZmqE6dOqVNmzZpypQpioyMVHBwsNq0aaO4uDhFR0dLcr7kl+e///2vOnToIHd3d918881KSkpyvLd69WrdeuutevbZZxUSEqJGjRqpd+/emjlzpqPN+PHj1aJFC7311lsKDAyUp6en7rvvPp06deoaHDUAACgLykyg8vb2lre3t1asWKHMzMwr3u7ZZ5/VyJEjtXv3bnXo0EF33XWXjh8/LunCjNaePXv03XffXbKPn376SYmJiVq1apXWrFmj5ORkPfHEE4W2z8zMVFpamtMLAACUX2UmULm6uiohIUGLFi1SpUqV1LFjR/3tb3/TN998c8ntnnzySfXp00eNGzfWnDlz5Ofnp/nz50uSnnrqKbVu3VrNmjVTnTp11L9/fy1YsCBfYDt37pwWLVqkFi1aqHPnzpo5c6aWLl2qw4cPF7jPSZMmyc/Pz/EKDAwsnkEAAADXpTITqKQL91D99ttvWrlypaKiopSUlKRWrVopISGh0G3at2/v+NnV1VXh4eHat2+fJMnLy0v/+te/9NNPP2nMmDHy9vbWyJEj1aZNG/3xxx+O7YKCglS7dm2nPnNzc/X9998XuM+4uDidPn3a8Tp48KDFIwcAANezMhWoJMnd3V3dunXTiy++qM2bNys2Nlbjxo27qj5sNpvTcv369TVkyBC988472rVrl/bu3asPPvjgsttf3E8eu90uX19fpxcAACi/ylyguliTJk2UkZFR6Ptbt251/Jydna2dO3cqNDS00PZ16tSRp6enU5+pqan67bffHMtbtmyRi4uLGjVqZLF6AABQHpSZj004fvy47rvvPg0aNEhhYWHy8fHRjh079Oqrr+ruu+8udLtZs2apYcOGaty4saZPn66TJ09q0KBBki48wffHH3+oZ8+eCg4O1qlTpzRjxgxlZWWpW7dujj7c3d0VExOj1157TWlpaRo+fLj69et31R/TAAAAyqcyE6i8vb3Vtm1bTZ8+Xfv371dWVpYCAwM1dOhQ/e1vfyt0u8mTJ2vKlCnavXu36tevr48//lhVq1aVJEVERGjWrFl6+OGHdeTIEVWuXFktW7bUZ599ppCQEEcfDRo00L333quePXvqxIkT6tmzp2bPnl3ixwwAAMoGmzHGlHYR17Px48drxYoVlr7CJi0t7cLTfiMS5WL3LL7ibgApk6NLuwQAwA0q7+/36dOnL3s/dJm/hwoAAKC0EagAAAAsIlBdxvjx4y1d7gMAAOUfgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARa6lXcCN5Lv4qMt+WzUAACh7mKECAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhPSr+Gmo5bKxe7Z2mXkU/K5OjSLgEAgDKNGSoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsKjUA9X48ePVokWLEuk7KSlJNptNp06dKrY+U1JSZLPZlJycXGx9AgCAsu2qAlVsbKxsNlu+V48ePUqqPgAAgOue69Vu0KNHDy1cuNBpnd1uL7aCiktWVlZplwAAAG4QV33Jz263KyAgwOlVuXJlSZLNZtNbb72lXr16ydPTU40bN9aWLVv0008/qUuXLvLy8lL79u21f//+fP2+9dZbCgwMlKenp+677z6ny3Tbt29Xt27dVLVqVfn5+SkiIkK7du1y2t5ms2nu3Lm6++675eXlpZdffjnfPs6ePavo6Gi1a9dOJ06ckCQtXLhQjRs3lru7u0JDQzV79mynbbZt26aWLVvK3d1d4eHh2r1799UOGQAAKOeK/R6qCRMm6OGHH1ZycrJCQ0M1YMAADRs2THFxcdqxY4ck6cknn3Ta5qefflJiYqJWrVqlNWvWKDk5WU888YTj/fT0dMXExGjjxo3aunWrGjZsqJ49eyo9Pd2pn3Hjxunuu+/Wt99+q0GDBjm9d/r0aXXv3l3nz5/XunXrVKVKFc2bN0+jR4/WK6+8on379mnixIkaO3asFi1aJEnKyMhQr169FBISop07d2r8+PEaNWpUcQ8ZAAAo4676kt/q1avl7e3ttO7555/X2LFjJUmPPPKI+vXr51jfvn17jR07VlFRUZKkp59+Wo888ojT9ufOndOiRYtUu3ZtSdLMmTMVHR2tadOmKSAgQLfddptT+7feekuVK1fWhg0b1KtXL8f6AQMGOAWpAwcOSJKOHDmi+++/X/Xr19f777+vihUrSroQ/qZNm6Z7771XklS3bl3t3btXb731lmJiYvTee+8pJydHCxYskKenp26++Wb9+uuveuyxxy45RpmZmcrMzHQsp6WlXbI9AAAo2646UEVGRmrOnDlO66pUqeL4OSwszPFzjRo1JEnNmjVzWnfu3DmlpaXJ19dXkhQUFOQIU5LUvn175ebm6vvvv1dAQICOHj2qF198UevXr9eRI0eUk5OjP/74Q6mpqU51hIeHF1jz7bffrtatWysxMVEVKlSQJB07dkwHDx7U4MGDNXToUEfb7Oxs+fn5SZL27dun5s2by9PT06m2y5k0aZLi4+Mv2w4AAJQPVx2ovLy81KBBg0Lfd3Nzc/xss9kKXZebm1toH3lt8v43NjZWx44d0+uvv67g4GDZ7Xa1b99e58+fz1dbQaKjo/XRRx9p7969jnCXt/958+apbdu2Tu3zQpcxptAaLyUuLk7PPPOMYzktLU2BgYFF6gsAAFz/rjpQlYTU1FT99ttvqlWrliRpy5YtcnFxUaNGjSRJGzdu1OzZs9WzZ09J0sGDB/X7779fcf+TJ0+Wt7e3unbtqqSkJDVp0kQ1atTQTTfdpJ9//lkPPvhggds1adJEixcv1tmzZ+Xh4SFJ2rp162X3Z7fbr8snHwEAQMm46kCVmZmpw4cPO3fi6qqqVasWuQh3d3fFxMTotddeU1pamoYPH65+/fopICBAktSgQQMtXrxY4eHhSktL07PPPusIOFfqtddeU05Ojm677TYlJSUpNDRU48eP1/Dhw+Xr66s77rhDmZmZ2rFjh06ePKlnnnlGAwYM0OjRozV48GCNGTNGKSkpeu2114p8nAAAoHy66qf81qxZo5o1azq9br31VktFNGjQQPfee6969uyp7t27q2nTpk4fX7BgwQKdPHlSLVu21EMPPaThw4erevXqV72f6dOnq1+/frrtttv0ww8/aMiQIXrnnXeUkJCgZs2aKSIiQgkJCapbt64kydvbW6tWrdLevXvVsmVLjR49WlOmTLF0rAAAoPyxmaLeKIQrlpaWJj8/PwWOSJSL3fPyG1xjKZOjS7sEAACuO3l/v0+fPu14kK4wpf5dfgAAAGUdgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALHIt7QJuJN/FR8nX17e0ywAAAMWMGSoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARXyw5zXUdNxaudg9S3QfKZOjS7R/AACQHzNUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAtVlpKSkyGazKTk5ubRLAQAA16kyF6hiY2Nls9lks9nk5uamevXqadSoUcrIyCjt0gAAwA3KtbQLKIoePXpo4cKFysrK0saNGzVkyBBlZGRozpw5V9WPMUY5OTlydS2TwwAAAK4TZW6GSpLsdrsCAgIUGBioAQMG6MEHH9SKFSv07rvvKjw8XD4+PgoICNCAAQN09OhRx3ZJSUmy2Wxau3atwsPDZbfbtXHjRuXm5mrKlClq0KCB7Ha7goKC9Morrzjt8+eff1ZkZKQ8PT3VvHlzbdmy5VofNgAAuE6VyUB1MQ8PD2VlZen8+fOaMGGCvv76a61YsUIHDhxQbGxsvvbPPfecJk2apH379iksLExxcXGaMmWKxo4dq71792rJkiWqUaOG0zajR4/WqFGjlJycrEaNGumBBx5QdnZ2gfVkZmYqLS3N6QUAAMqvMn+ta9u2bVqyZIm6du2qQYMGOdbXq1dPM2bMUJs2bXTmzBl5e3s73nvppZfUrVs3SVJ6erreeOMNvfnmm4qJiZEk1a9fX7feeqvTfkaNGqXo6GhJUnx8vG6++Wb99NNPCg0NzVfTpEmTFB8fX+zHCgAArk9lcoZq9erV8vb2lru7u9q3b6/OnTtr5syZ2r17t+6++24FBwfLx8dHXbp0kSSlpqY6bR8eHu74ed++fcrMzFTXrl0vuc+wsDDHzzVr1pQkp8uJfxYXF6fTp087XgcPHizKYQIAgDKiTM5QRUZGas6cOXJzc1OtWrXk5uamjIwMde/eXd27d9e7776ratWqKTU1VVFRUTp//rzT9l5eXo6fPTw8rmifbm5ujp9tNpskKTc3t8C2drtddrv9ag8LAACUUWVyhsrLy0sNGjRQcHCwI+j897//1e+//67JkyerU6dOCg0NLXQG6c8aNmwoDw8PrVu3rqTLBgAA5VSZnKEqSFBQkCpWrKiZM2fq0Ucf1XfffacJEyZcdjt3d3c9//zzeu6551SxYkV17NhRx44d0549ezR48OBrUDkAACjryuQMVUGqVaumhIQELVu2TE2aNNHkyZP12muvXdG2Y8eO1ciRI/Xiiy+qcePGuv/++69odgsAAECSbMYYU9pFlHdpaWny8/NT4IhEudg9S3RfKZOjS7R/AABuFHl/v0+fPi1fX99Lti03M1QAAAClhUAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAItcS7uAG8l38VGX/bZqAABQ9jBDBQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABY5FraBdwIjDGSpLS0tFKuBAAAXKm8v9t5f8cvhUB1DRw/flySFBgYWMqVAACAq5Weni4/P79LtiFQXQNVqlSRJKWmpl72FwJnaWlpCgwM1MGDB+Xr61va5ZQpjF3RMXZFx9gVHWNXNCU5bsYYpaenq1atWpdtS6C6BlxcLtyq5ufnxz+SIvL19WXsioixKzrGrugYu6Jj7IqmpMbtSidCuCkdAADAIgIVAACARQSqa8But2vcuHGy2+2lXUqZw9gVHWNXdIxd0TF2RcfYFc31Mm42cyXPAgIAAKBQzFABAABYRKACAACwiEAFAABgEYEKAADAIgJVEc2ePVt169aVu7u7brnlFm3cuPGS7Tds2KBbbrlF7u7uqlevnubOnZuvzUcffaQmTZrIbrerSZMm+uc//1lS5Zea4h63hIQE2Wy2fK9z586V5GGUiqsZu0OHDmnAgAEKCQmRi4uLRowYUWC7G+Gck4p/7DjvCrZ8+XJ169ZN1apVk6+vr9q3b6+1a9fma8d5l9+VjB3nXcE2bdqkjh07yt/fXx4eHgoNDdX06dPztSvx887gqi1dutS4ubmZefPmmb1795qnn37aeHl5mV9++aXA9j///LPx9PQ0Tz/9tNm7d6+ZN2+ecXNzMx9++KGjzebNm02FChXMxIkTzb59+8zEiRONq6ur2bp167U6rBJXEuO2cOFC4+vraw4dOuT0Km+uduwOHDhghg8fbhYtWmRatGhhnn766XxtboRzzpiSGTvOu4LH7umnnzZTpkwx27ZtMz/88IOJi4szbm5uZteuXY42nHdFHzvOu4LHbteuXWbJkiXmu+++MwcOHDCLFy82np6e5q233nK0uRbnHYGqCNq0aWMeffRRp3WhoaHmhRdeKLD9c889Z0JDQ53WDRs2zLRr186x3K9fP9OjRw+nNlFRUaZ///7FVHXpK4lxW7hwofHz8yv2Wq83Vzt2fxYREVFgKLgRzjljSmbsOO8uP3Z5mjRpYuLj4x3LnHdFHzvOuysfu3vuuccMHDjQsXwtzjsu+V2l8+fPa+fOnerevbvT+u7du2vz5s0FbrNly5Z87aOiorRjxw5lZWVdsk1hfZY1JTVuknTmzBkFBwerdu3a6tWrl3bv3l38B1CKijJ2V6K8n3NSyY2dxHl3JXJzc5Wenu74gniJ887K2Emcd1di9+7d2rx5syIiIhzrrsV5R6C6Sr///rtycnJUo0YNp/U1atTQ4cOHC9zm8OHDBbbPzs7W77//fsk2hfVZ1pTUuIWGhiohIUErV67U+++/L3d3d3Xs2FE//vhjyRxIKSjK2F2J8n7OSSU3dpx3VzZ206ZNU0ZGhvr16+dYx3lX9LHjvLv02NWuXVt2u13h4eF64oknNGTIEMd71+K8cy22nm4wNpvNadkYk2/d5dpfvP5q+yyLinvc2rVrp3bt2jne79ixo1q1aqWZM2dqxowZxVX2daEkzo8b4ZyTiv84Oe8uP3bvv/++xo8fr48//ljVq1cvlj7LmuIeO867S4/dxo0bdebMGW3dulUvvPCCGjRooAceeMBSn1eDQHWVqlatqgoVKuRLtUePHs2XfvMEBAQU2N7V1VX+/v6XbFNYn2VNSY3bxVxcXNS6dety9V9sRRm7K1Hezzmp5MbuYpx3zj744AMNHjxYy5Yt0+233+70Hudd0cfuYpx3zurWrStJatasmY4cOaLx48c7AtW1OO+45HeVKlasqFtuuUWff/650/rPP/9cHTp0KHCb9u3b52v/2WefKTw8XG5ubpdsU1ifZU1JjdvFjDFKTk5WzZo1i6fw60BRxu5KlPdzTiq5sbsY593/9/777ys2NlZLlixRdHR0vvc574o+dhfjvCucMUaZmZmO5Wty3hXb7e03kLxHOufPn2/27t1rRowYYby8vExKSooxxpgXXnjBPPTQQ472eY////WvfzV79+418+fPz/f4/3/+8x9ToUIFM3nyZLNv3z4zefLkcvcocUmM2/jx482aNWvM/v37ze7du80jjzxiXF1dzVdffXXNj68kXe3YGWPM7t27ze7du80tt9xiBgwYYHbv3m327NnjeP9GOOeMKZmx47wreOyWLFliXF1dzaxZs5we6z916pSjDedd0ceO867gsXvzzTfNypUrzQ8//GB++OEHs2DBAuPr62tGjx7taHMtzjsCVRHNmjXLBAcHm4oVK5pWrVqZDRs2ON6LiYkxERERTu2TkpJMy5YtTcWKFU2dOnXMnDlz8vW5bNkyExISYtzc3ExoaKj56KOPSvowrrniHrcRI0aYoKAgU7FiRVOtWjXTvXt3s3nz5mtxKNfc1Y6dpHyv4OBgpzY3wjlnTPGPHefdBRePXURERIFjFxMT49Qn513Rxo7z7oKLx27GjBnm5ptvNp6ensbX19e0bNnSzJ492+Tk5Dj1WdLnnc2Y/7vLFwAAAEXCPVQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAMqM2NhY9e7du7TLKFBKSopsNpuSk5NLuxQApYBABQAWnT9/vrRLAFDKCFQAyqQuXbroqaee0ogRI1S5cmXVqFFDb7/9tjIyMvTII4/Ix8dH9evX16effurYJikpSTabTf/617/UvHlzubu7q23btvr222+d+v7oo4908803y263q06dOpo2bZrT+3Xq1NHLL7+s2NhY+fn5aejQoapbt64kqWXLlrLZbOrSpYskafv27erWrZuqVq0qPz8/RUREaNeuXU792Ww2vfPOO7rnnnvk6emphg0bauXKlU5t9uzZo+joaPn6+srHx0edOnXS/v37He8vXLhQjRs3lru7u0JDQzV79mzLYwzgyhGoAJRZixYtUtWqVbVt2zY99dRTeuyxx3TfffepQ4cO2rVrl6KiovTQQw/pjz/+cNru2Wef1Wuvvabt27erevXquuuuu5SVlSVJ2rlzp/r166f+/fvr22+/1fjx4zV27FglJCQ49TF16lQ1bdpUO3fu1NixY7Vt2zZJ0hdffKFDhw5p+fLlkqT09HTFxMRo48aN2rp1qxo2bKiePXsqPT3dqb/4+Hj169dP33zzjXr27KkHH3xQJ06ckCT973//U+fOneXu7q7169dr586dGjRokLKzsyVJ8+bN0+jRo/XKK69o3759mjhxosaOHatFixYV+5gDKESxftUyAJSgmJgYc/fddxtjjImIiDC33nqr473s7Gzj5eVlHnroIce6Q4cOGUlmy5YtxhhjvvzySyPJLF261NHm+PHjxsPDw3zwwQfGGGMGDBhgunXr5rTfZ5991jRp0sSxHBwcbHr37u3U5sCBA0aS2b179yWPITs72/j4+JhVq1Y51kkyY8aMcSyfOXPG2Gw28+mnnxpjjImLizN169Y158+fL7DPwMBAs2TJEqd1EyZMMO3bt79kLQCKDzNUAMqssLAwx88VKlSQv7+/mjVr5lhXo0YNSdLRo0edtmvfvr3j5ypVqigkJET79u2TJO3bt08dO3Z0at+xY0f9+OOPysnJcawLDw+/ohqPHj2qRx99VI0aNZKfn5/8/Px05swZpaamFnosXl5e8vHxcdSdnJysTp06yc3NLV//x44d08GDBzV48GB5e3s7Xi+//LLTJUEAJcu1tAsAgKK6OGDYbDandTabTZKUm5t72b7y2hpjHD/nMcbka+/l5XVFNcbGxurYsWN6/fXXFRwcLLvdrvbt2+e7kb2gY8mr28PDo9D+89rMmzdPbdu2dXqvQoUKV1QjAOsIVABuOFu3blVQUJAk6eTJk/rhhx8UGhoqSWrSpIk2bdrk1H7z5s1q1KjRJQNKxYoVJclpFkuSNm7cqNmzZ6tnz56SpIMHD+r333+/qnrDwsK0aNEiZWVl5QteNWrU0E033aSff/5ZDz744FX1C6D4EKgA3HBeeukl+fv7q0aNGho9erSqVq3q+HyrkSNHqnXr1powYYLuv/9+bdmyRW+++eZln5qrXr26PDw8tGbNGtWuXVvu7u7y8/NTgwYNtHjxYoWHhystLU3PPvvsJWecCvLkk09q5syZ6t+/v+Li4uTn56etW7eqTZs2CgkJ0fjx4zV8+HD5+vrqjjvuUGZmpnbs2KGTJ0/qmWeeKeowAbgK3EMF4IYzefJkPf3007rlllt06NAhrVy50jHD1KpVKyUmJmrp0qVq2rSpXnzxRb300kuKjY29ZJ+urq6aMWOG3nrrLdWqVUt33323JGnBggU6efKkWrZsqYceekjDhw9X9erVr6pef39/rV+/XmfOnFFERIRuueUWzZs3zzFbNWTIEL3zzjtKSEhQs2bNFBERoYSEBMdHOQAoeTZT0M0BAFAOJSUlKTIyUidPnlSlSpVKuxwA5QgzVAAAABYRqAAAACzikh8AAIBFzFABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWPT/AL5387v/WEZ9AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate average feature importance across all trees\n",
    "importances = np.mean([tree.feature_importances_ for tree in bagging_model.estimators_], axis=0)\n",
    "\n",
    "# Sort features by importance\n",
    "indices = np.argsort(importances)\n",
    "sorted_features = X_train.columns[indices]\n",
    "sorted_importances = importances[indices]\n",
    "\n",
    "# Plot\n",
    "plt.barh(sorted_features, sorted_importances)\n",
    "plt.xlabel('Importance')\n",
    "plt.title('Feature Importances in Bagging')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1cc2eb0",
   "metadata": {},
   "source": [
    "From our bar chart, it looks like `Sex`, `Age`, and `Fare` played the most important role in predicting survival. Now, we do the same modeling with Random Forest and AdaBoost. \n",
    "\n",
    "## Random Forest\n",
    "\n",
    "We implement Random Forests using scikit-learn's `RandomForestClassifier`. See [here](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html) for documentation. \n",
    "\n",
    "We set the following parameters:\n",
    "- **`n_estimators`**: The number of trees in the forest. We use 100 trees, same as in bagging.\n",
    "- **`random_state`**: To ensures reproducibility.\n",
    "\n",
    "By default, `RandomForestClassifier` uses $m = \\sqrt{p}$ features at each split for classification, where $p$ is the total number of features. With our 7 features, $\\sqrt{7} \\approx 2.65$, so 2 features are randomly selected at each split."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aea25566",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Accuracy: 0.8112\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Create a Random Forest classifier\n",
    "rf_model = RandomForestClassifier(\n",
    "    n_estimators=100,\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "# Train the model\n",
    "rf_model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_pred_rf = rf_model.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "rf_accuracy = accuracy_score(y_test, y_pred_rf)\n",
    "print(f\"Random Forest Accuracy: {rf_accuracy:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48e8b258",
   "metadata": {},
   "source": [
    "Our Random Forest model achieves approximately 81% accuracy, which is a slight improvement over the Bagging model (80%). This demonstrates how the additional randomness of feature selection can lead to better predictions. Lets compare the importance of the features in random forest model with bagging. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "38c7883a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAHFCAYAAAA0SmdSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEq0lEQVR4nO3deXyM5/7/8fdkm8kiQZBQEXtQYmnQUFKUIFqOqiotqeVUN3UsbWMpqkVKj5Zj6VFEdbG0jtPqelriy0Gt0RK0ttJjqaJCEIlcvz/6y7Qjie0OI/F6Ph7zkPue677uz33NLfPOvczYjDFGAAAAuG4e7i4AAACgsCNQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUKHQS0pKks1my/MxZMiQG7LO1NRUjR49Wvv3778h/Vuxf/9+2Ww2TZo0yd2lXLc1a9Zo9OjR+u2339xdyk0THx+vihUrFlh/OftBzsPDw0MlSpRQq1at9NVXXxXYeq5VcnKybDabkpOT3VbDpfL7/VGqVCl3l5anzz77TKNHj3Z3GbiEl7sLAArK3LlzVaNGDZd55cqVuyHrSk1N1ZgxY3TvvfcW6JsgfrdmzRqNGTNG8fHxKl68uLvLuSlGjhyp5557rsD7ffbZZ9W9e3ddvHhRO3fu1JgxY9S+fXstX75czZs3L/D1FVZdunTR4MGDXeZ5e3u7qZrL++yzzzRt2jRC1S2GQIUio3bt2oqKinJ3GZZkZmbKZrPJy+v2/K957tw5ORwOd5fhFlWqVLkh/VaoUEF33323JKlp06aqVq2aYmJiNHv2bALVn4SEhDjHqSBdvHhRWVlZstvtBd43bi2c8sNtY+HChYqOjpa/v78CAgIUGxurLVu2uLTZuHGjunXrpooVK8rX11cVK1bUI488op9++snZJikpSQ899JAkqUWLFs7TA0lJSZKkihUrKj4+Ptf67733Xt17773O6ZxTH/Pnz9fgwYN1xx13yG63a/fu3ZKkr7/+Wq1atVJgYKD8/PzUtGlTffPNN9e17TmnRZcvX65+/fopODhYgYGB6tmzp9LT03XkyBF17dpVxYsXV9myZTVkyBBlZmY6l885ffTaa6/p1VdfVYUKFeRwOBQVFZVnTatXr1arVq1UrFgx+fn5qUmTJvr000/zrOmrr75S7969Vbp0afn5+SkhIUFDhw6VJFWqVMk5vjmniBYuXKg2bdqobNmy8vX1Vc2aNfXiiy8qPT3dpf/4+HgFBARo9+7dat++vQICAhQWFqbBgwcrIyPDpW1GRoZefvll1axZUw6HQ8HBwWrRooXWrFnjbGOM0fTp01WvXj35+vqqRIkS6tKli/bu3evS15YtW9ShQweVKVNGdrtd5cqVU1xcnH7++efLvkZ5nfKz2Wx65plnNH/+fNWsWVN+fn6qW7euli1bdtm+Lifnj46jR4+6zJ82bZqaN2+uMmXKyN/fX3Xq1NFrr73msh9Iv+/HtWvX1oYNG9SsWTP5+fmpcuXKmjBhgrKzs13a7ty5U23btpWfn59KlSql/v376/Tp03nWNWfOHNWtW1cOh0MlS5bUX/7yF+3YscOlTc5runPnTsXGxsrf319ly5bVhAkTJEnr1q3TPffcI39/f1WvXl3z5s277nG61IEDB/Too486X9eaNWvq9ddfd9nmP/8/eeWVV1SpUiXZ7XatWLFC0u+/Xx544AGVLFlSDodD9evX16JFi1zWc/bsWQ0ZMkSVKlVyjkVUVJQ++OAD5xhMmzZNkuupylvx8oPbze35ZzCKpJy/BP8s50jPuHHjNGLECD3++OMaMWKELly4oIkTJ6pZs2Zav369atWqJen3X4gRERHq1q2bSpYsqcOHD2vGjBlq2LChUlNTVapUKcXFxWncuHEaNmyYpk2bpgYNGki6/iMMCQkJio6O1syZM+Xh4aEyZcro3XffVc+ePdWxY0fNmzdP3t7eeuuttxQbG6svv/xSrVq1uq519e3bV507d9aCBQu0ZcsWDRs2TFlZWdq1a5c6d+6sv/71r/r666+VmJiocuXKadCgQS7L/+Mf/1B4eLjeeOMNZWdn67XXXlO7du20cuVKRUdHS5JWrlyp1q1bKzIyUrNnz5bdbtf06dN1//3364MPPtDDDz/s0mfv3r0VFxen+fPnKz09XVFRUTp79qymTp2qJUuWqGzZspLkfI1+/PFHtW/fXgMHDpS/v7927typxMRErV+/XsuXL3fpOzMzUw888ID69OmjwYMH6//+7/80duxYBQUF6aWXXpIkZWVlqV27dlq1apUGDhyoli1bKisrS+vWrdOBAwfUpEkTSdITTzyhpKQkDRgwQImJiTpx4oRefvllNWnSRFu3blVISIjS09PVunVrVapUSdOmTVNISIiOHDmiFStW5BskruTTTz/Vhg0b9PLLLysgIECvvfaa/vKXv2jXrl2qXLnyNfe3b98+SVL16tVd5u/Zs0fdu3dXpUqV5OPjo61bt+rVV1/Vzp07NWfOHJe2R44cUY8ePTR48GCNGjVK//rXv5SQkKBy5cqpZ8+ekn4PbDExMfL29tb06dMVEhKi9957T88880yumsaPH69hw4bpkUce0fjx43X8+HGNHj1a0dHR2rBhg6pVq+Zsm5mZqc6dO6t///4aOnSo3n//fSUkJCgtLU0fffSRXnjhBZUvX15Tp05VfHy8ateurbvuuuuK42KMyfX7w9PTUzabTceOHVOTJk104cIFjR07VhUrVtSyZcs0ZMgQ7dmzR9OnT3dZbsqUKapevbomTZqkwMBAVatWTStWrFDbtm3VuHFjzZw5U0FBQVqwYIEefvhhnT171vlH2KBBgzR//ny98sorql+/vtLT07Vt2zYdP35c0u+nhtPT0/Xhhx9q7dq1znXm/D+BGxmgkJs7d66RlOcjMzPTHDhwwHh5eZlnn33WZbnTp0+b0NBQ07Vr13z7zsrKMmfOnDH+/v7mzTffdM5fvHixkWRWrFiRa5nw8HDTq1evXPNjYmJMTEyMc3rFihVGkmnevLlLu/T0dFOyZElz//33u8y/ePGiqVu3rmnUqNFlRsOYffv2GUlm4sSJznk5Y3TpGHTq1MlIMn//+99d5terV880aNAgV5/lypUz586dc85PS0szJUuWNPfdd59z3t13323KlCljTp8+7ZyXlZVlateubcqXL2+ys7NdaurZs2eubZg4caKRZPbt23fZbc3OzjaZmZlm5cqVRpLZunWr87levXoZSWbRokUuy7Rv395EREQ4p9955x0jycyaNSvf9axdu9ZIMq+//rrL/IMHDxpfX1/z/PPPG2OM2bhxo5Fkli5detm689KrVy8THh7uMk+SCQkJMWlpac55R44cMR4eHmb8+PGX7S/nNUtMTDSZmZnm/PnzJiUlxURHR5uyZctedmwvXrxoMjMzzTvvvGM8PT3NiRMnnM/FxMQYSebbb791WaZWrVomNjbWOf3CCy8Ym81mUlJSXNq1bt3a5f/OyZMnja+vr2nfvr1LuwMHDhi73W66d+/uMkaSzEcffeScl5mZaUqXLm0kmc2bNzvnHz9+3Hh6eppBgwZddpyMMfn+/sjZJ1588cU8t/nJJ580NpvN7Nq1yxjzx5hXqVLFXLhwwaVtjRo1TP369U1mZqbL/A4dOpiyZcuaixcvGmOMqV27tunUqdNl63366acNb9+3Hk75och45513tGHDBpeHl5eXvvzyS2VlZalnz57KyspyPhwOh2JiYlzuNjpz5oxeeOEFVa1aVV5eXvLy8lJAQIDS09NznX4oKA8++KDL9Jo1a3TixAn16tXLpd7s7Gy1bdtWGzZsyHV662p16NDBZbpmzZqSpLi4uFzz/3yaM0fnzp1drnEqVqyY7r//fv3f//2fLl68qPT0dH377bfq0qWLAgICnO08PT312GOP6eeff9auXbsuu/1XsnfvXnXv3l2hoaHy9PSUt7e3YmJiJCnXa2Sz2XT//fe7zIuMjHTZts8//1wOh0O9e/fOd53Lli2TzWbTo48+6vKahIaGqm7dus59qGrVqipRooReeOEFzZw5U6mpqde0bXlp0aKFihUr5pwOCQlRmTJl8nx98vLCCy/I29tbDodD9erV07Zt2/TJJ5/kOr24ZcsWPfDAAwoODnaOa8+ePXXx4kX98MMPLm1DQ0PVqFEjl3mXjuuKFSt05513qm7dui7tunfv7jK9du1anTt3Ltdp8rCwMLVs2TLXKWWbzab27ds7p728vFS1alWVLVtW9evXd84vWbLkNY1T165dc/3+6NSpkyRp+fLlqlWrVq5tjo+PlzEm15HRBx54wOWC9t27d2vnzp3q0aOHJLnsQ+3bt9fhw4ed/y8aNWqkzz//XC+++KKSk5N17ty5q6of7scpPxQZNWvWzPOi9JxrRRo2bJjnch4ef/xd0b17d33zzTcaOXKkGjZsqMDAQOcv8Bv1i+3SQ/U59Xbp0iXfZU6cOCF/f/9rXlfJkiVdpn18fPKdf/78+VzLh4aG5jnvwoULOnPmjE6fPi1jTJ6nH3LuuMw5dZHjWk5VnDlzRs2aNZPD4dArr7yi6tWry8/PTwcPHlTnzp1zvUZ+fn65LnK32+0u23bs2DGVK1fOZT+41NGjR2WMUUhISJ7P55x6CwoK0sqVK/Xqq69q2LBhOnnypMqWLat+/fppxIgR13XXWHBwcK55drv9qvfH5557To8++qgyMjK0bt06jRgxQh07dtTWrVudfR84cEDNmjVTRESE3nzzTVWsWFEOh0Pr16/X008/nWtdV1PT8ePHValSpVztLt2HcvaH/PaZ//znPy7z8npNfXx8cu3DOfPz2o/zUrp06Xxvajl+/Hied/Ne7T6d8396yJAh+X6Uy6+//irp99OF5cuX18KFC5WYmCiHw6HY2FhNnDjR5dQnbj0EKhR5OZ8l8+GHHyo8PDzfdqdOndKyZcs0atQovfjii875GRkZOnHixFWvz+Fw5LroWfr9F2Zen2tjs9nyrHfq1Kn53nWU3xv7jXbkyJE85/n4+CggIEBeXl7y8PDQ4cOHc7U7dOiQJOUag0u3/3KWL1+uQ4cOKTk52XlUSpKlz6sqXbq0Vq9erezs7HxDValSpWSz2bRq1ao879b687w6depowYIFMsbou+++U1JSkl5++WX5+vq67Fc3S/ny5Z1BoWnTpgoNDdWjjz6qUaNG6R//+IckaenSpUpPT9eSJUtc/o+kpKRc93qDg4Pz3V8ubScp333mVvgsqODgYEv7dM7zCQkJ6ty5c57riIiIkCT5+/trzJgxGjNmjI4ePeo8WnX//fdr586dlrcFNw6n/FDkxcbGysvLS3v27FFUVFSeD+n3X4LGmFxvmG+//bYuXrzoMi+nTV5HCSpWrKjvvvvOZd4PP/yQ61RXfpo2barixYsrNTU133pzjizdbEuWLHH5i//06dP65JNP1KxZM3l6esrf31+NGzfWkiVLXMYmOztb7777rsqXL5/rYui85De+OW9Ul75Gb7311nVvU7t27XT+/HnnXZp56dChg4wx+t///pfn61GnTp1cy9hsNtWtW1eTJ09W8eLFtXnz5uuusSD16NFD9957r2bNmuU8HZbXuBpjNGvWrOteT4sWLbR9+3Zt3brVZf7777/vMh0dHS1fX1+9++67LvN//vlnLV++/LpvwChIrVq1Umpqaq7X8J133pHNZlOLFi0uu3xERISqVaumrVu35vt/+s+ndXOEhIQoPj5ejzzyiHbt2qWzZ89KuvzvH7gPR6hQ5FWsWFEvv/yyhg8frr1796pt27YqUaKEjh49qvXr1zv/IgwMDFTz5s01ceJElSpVShUrVtTKlSs1e/bsXB8uWbt2bUnSP//5TxUrVkwOh0OVKlVScHCwHnvsMT366KN66qmn9OCDD+qnn37Sa6+9ptKlS19VvQEBAZo6dap69eqlEydOqEuXLipTpoyOHTumrVu36tixY5oxY0ZBD9NV8fT0VOvWrTVo0CBlZ2crMTFRaWlpGjNmjLPN+PHj1bp1a7Vo0UJDhgyRj4+Ppk+frm3btumDDz64qiNSOQHlzTffVK9eveTt7a2IiAg1adJEJUqUUP/+/TVq1Ch5e3vrvffey/WmfS0eeeQRzZ07V/3799euXbvUokULZWdn69tvv1XNmjXVrVs3NW3aVH/961/1+OOPa+PGjWrevLn8/f11+PBhrV69WnXq1NGTTz6pZcuWafr06erUqZMqV64sY4yWLFmi3377Ta1bt77uGgtaYmKiGjdurLFjx+rtt99W69at5ePjo0ceeUTPP/+8zp8/rxkzZujkyZPXvY6BAwdqzpw5iouL0yuvvOK8y+/SoyzFixfXyJEjNWzYMPXs2VOPPPKIjh8/rjFjxsjhcGjUqFFWN9eyv/3tb3rnnXcUFxenl19+WeHh4fr00081ffp0Pfnkk1f1R8Jbb72ldu3aKTY2VvHx8brjjjt04sQJ7dixQ5s3b9bixYslSY0bN1aHDh0UGRmpEiVKaMeOHZo/f76io6Pl5+cn6Y//H4mJiWrXrp08PT0VGRnptj+08P+573p4oGDk3C22YcOGy7ZbunSpadGihQkMDDR2u92Eh4ebLl26mK+//trZ5ueffzYPPvigKVGihClWrJhp27at2bZtW5537r3xxhumUqVKxtPT00gyc+fONcb8fufZa6+9ZipXrmwcDoeJiooyy5cvz/cuv8WLF+dZ78qVK01cXJwpWbKk8fb2NnfccYeJi4vLt32Oy93ld+kYjRo1ykgyx44dc5nfq1cv4+/vn6vPxMREM2bMGFO+fHnj4+Nj6tevb7788stcNaxatcq0bNnS+Pv7G19fX3P33XebTz75xKXNlV63hIQEU65cOePh4eFyV9iaNWtMdHS08fPzM6VLlzZ9+/Y1mzdvdnkN8tqGS7f5z86dO2deeuklU61aNePj42OCg4NNy5YtzZo1a1zazZkzxzRu3Ni5XVWqVDE9e/Y0GzduNMYYs3PnTvPII4+YKlWqGF9fXxMUFGQaNWpkkpKS8tzGP8vvLr+nn346V9v87iT9s7z2gz976KGHjJeXl9m9e7cxxphPPvnE1K1b1zgcDnPHHXeYoUOHms8//zzX3awxMTHmzjvvvKr6U1NTTevWrY3D4TAlS5Y0ffr0Mf/+97/zvEP27bffNpGRkcbHx8cEBQWZjh07mu3bt+daR16vaX41hYeHm7i4uDy3/8/yG+c/++mnn0z37t1NcHCw8fb2NhEREWbixInOu/OMufKYb9261XTt2tWUKVPGeHt7m9DQUNOyZUszc+ZMZ5sXX3zRREVFmRIlShi73W4qV65s/va3v5lff/3V2SYjI8P07dvXlC5d2thstqu6IxY3ns0YY252iANQuOzfv1+VKlXSxIkTb9j3IwJAYcY1VAAAABYRqAAAACzilB8AAIBFHKECAACwiEAFAABgEYEKAADAIj7Y8ybIzs7WoUOHVKxYsWv6mg0AAOA+xhidPn36it/3KRGobopDhw4pLCzM3WUAAIDrcPDgQZUvX/6ybQhUN0HOdzQdPHhQgYGBbq4GAABcjbS0NIWFheX5XYuXIlDdBDmn+QIDAwlUAAAUMldzuQ4XpQMAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAi7zcXcDtpPaoL+Vh93N3GQAAFCn7J8S5uwSOUAEAAFhFoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAi26LQLVmzRp5enqqbdu27i4FAAAUQbdFoJozZ46effZZrV69WgcOHHB3OQAAoIgp8oEqPT1dixYt0pNPPqkOHTooKSnJ5fmPP/5Y1apVk6+vr1q0aKF58+bJZrPpt99+c7ZZs2aNmjdvLl9fX4WFhWnAgAFKT0+/uRsCAABuWUU+UC1cuFARERGKiIjQo48+qrlz58oYI0nav3+/unTpok6dOiklJUVPPPGEhg8f7rL8999/r9jYWHXu3FnfffedFi5cqNWrV+uZZ55xx+YAAIBbUJEPVLNnz9ajjz4qSWrbtq3OnDmjb775RpI0c+ZMRUREaOLEiYqIiFC3bt0UHx/vsvzEiRPVvXt3DRw4UNWqVVOTJk00ZcoUvfPOOzp//nye68zIyFBaWprLAwAAFF1FOlDt2rVL69evV7du3SRJXl5eevjhhzVnzhzn8w0bNnRZplGjRi7TmzZtUlJSkgICApyP2NhYZWdna9++fXmud/z48QoKCnI+wsLCbsDWAQCAW4WXuwu4kWbPnq2srCzdcccdznnGGHl7e+vkyZMyxshms7ksk3M6MEd2draeeOIJDRgwIFf/FSpUyHO9CQkJGjRokHM6LS2NUAUAQBFWZANVVlaW3nnnHb3++utq06aNy3MPPvig3nvvPdWoUUOfffaZy3MbN250mW7QoIG2b9+uqlWrXvW67Xa77Hb79RcPAAAKlSIbqJYtW6aTJ0+qT58+CgoKcnmuS5cumj17tpYsWaK///3veuGFF9SnTx+lpKQ47wLMOXL1wgsv6O6779bTTz+tfv36yd/fXzt27NB//vMfTZ069WZvFgAAuAUV2WuoZs+erfvuuy9XmJJ+P0KVkpKikydP6sMPP9SSJUsUGRmpGTNmOO/yyznCFBkZqZUrV+rHH39Us2bNVL9+fY0cOVJly5a9qdsDAABuXTZz6UVDt7lXX31VM2fO1MGDBwusz7S0tN8vTh+4SB52vwLrFwAASPsnxN2QfnPev0+dOqXAwMDLti2yp/yu1vTp09WwYUMFBwfrv//9ryZOnMhnTAEAgGty2weqH3/8Ua+88opOnDihChUqaPDgwUpISHB3WQAAoBC57QPV5MmTNXnyZHeXAQAACrEie1E6AADAzUKgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALDotv/qmZtp25jYK35bNQAAKHw4QgUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEZ+UfhPVHvWlPOx+7i4DAIBrtn9CnLtLuKVxhAoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsOi2CFS//PKLnnjiCVWoUEF2u12hoaGKjY3V2rVr3V0aAAAoArzcXcDN8OCDDyozM1Pz5s1T5cqVdfToUX3zzTc6ceKEu0sDAABFQJE/QvXbb79p9erVSkxMVIsWLRQeHq5GjRopISFBcXFxkqRTp07pr3/9q8qUKaPAwEC1bNlSW7dulSQdO3ZMoaGhGjdunLPPb7/9Vj4+Pvrqq6/csk0AAODWUuQDVUBAgAICArR06VJlZGTket4Yo7i4OB05ckSfffaZNm3apAYNGqhVq1Y6ceKESpcurTlz5mj06NHauHGjzpw5o0cffVRPPfWU2rRpk+c6MzIylJaW5vIAAABFV5EPVF5eXkpKStK8efNUvHhxNW3aVMOGDdN3330nSVqxYoW+//57LV68WFFRUapWrZomTZqk4sWL68MPP5QktW/fXv369VOPHj3Uv39/ORwOTZgwId91jh8/XkFBQc5HWFjYTdlWAADgHkU+UEm/X0N16NAhffzxx4qNjVVycrIaNGigpKQkbdq0SWfOnFFwcLDzaFZAQID27dunPXv2OPuYNGmSsrKytGjRIr333ntyOBz5ri8hIUGnTp1yPg4ePHgzNhMAALjJbXFRuiQ5HA61bt1arVu31ksvvaS+fftq1KhReuqpp1S2bFklJyfnWqZ48eLOn/fu3atDhw4pOztbP/30kyIjI/Ndl91ul91uvwFbAQAAbkW3TaC6VK1atbR06VI1aNBAR44ckZeXlypWrJhn2wsXLqhHjx56+OGHVaNGDfXp00fff/+9QkJCbm7RAADgllTkT/kdP35cLVu21LvvvqvvvvtO+/bt0+LFi/Xaa6+pY8eOuu+++xQdHa1OnTrpyy+/1P79+7VmzRqNGDFCGzdulCQNHz5cp06d0pQpU/T888+rZs2a6tOnj5u3DAAA3CqK/BGqgIAANW7cWJMnT9aePXuUmZmpsLAw9evXT8OGDZPNZtNnn32m4cOHq3fv3s6PSWjevLlCQkKUnJysN954QytWrFBgYKAkaf78+YqMjNSMGTP05JNPunkLAQCAu9mMMcbdRRR1aWlpv9/tN3CRPOx+7i4HAIBrtn9CnLtLuOly3r9PnTrlPKiSnyJ/yg8AAOBGI1ABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCIvdxdwO9k2JvaK31YNAAAKH45QAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFjEJ6XfRLVHfSkPu5+7ywAA3Kb2T4hzdwlFFkeoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFRS5QxcfHy2az5Xrs3r3b3aUBAIAiysvdBdwIbdu21dy5c13mlS5d+pr6uHjxomw2mzw8ilzmBAAABaxIpgW73a7Q0FCXx5tvvqk6derI399fYWFheuqpp3TmzBnnMklJSSpevLiWLVumWrVqyW6366efftKFCxf0/PPP64477pC/v78aN26s5ORk920cAAC45RTJQJUXDw8PTZkyRdu2bdO8efO0fPlyPf/88y5tzp49q/Hjx+vtt9/W9u3bVaZMGT3++OP673//qwULFui7777TQw89pLZt2+rHH39005YAAIBbTZE85bds2TIFBAQ4p9u1a6fFixc7pytVqqSxY8fqySef1PTp053zMzMzNX36dNWtW1eStGfPHn3wwQf6+eefVa5cOUnSkCFD9MUXX2ju3LkaN25cnuvPyMhQRkaGczotLa1Atw8AANxaimSgatGihWbMmOGc9vf314oVKzRu3DilpqYqLS1NWVlZOn/+vNLT0+Xv7y9J8vHxUWRkpHO5zZs3yxij6tWru/SfkZGh4ODgfNc/fvx4jRkzpoC3CgAA3KqKZKDy9/dX1apVndM//fST2rdvr/79+2vs2LEqWbKkVq9erT59+igzM9PZztfXVzabzTmdnZ0tT09Pbdq0SZ6eni7r+PMRsEslJCRo0KBBzum0tDSFhYUVxKYBAIBbUJEMVJfauHGjsrKy9Prrrzvv2lu0aNEVl6tfv74uXryoX375Rc2aNbvq9dntdtnt9uuuFwAAFC63xUXpVapUUVZWlqZOnaq9e/dq/vz5mjlz5hWXq169unr06KGePXtqyZIl2rdvnzZs2KDExER99tlnN6FyAABQGNwWgapevXr6+9//rsTERNWuXVvvvfeexo8ff1XLzp07Vz179tTgwYMVERGhBx54QN9++y2n8AAAgJPNGGPcXURRl5aWpqCgIIUNXCQPu5+7ywEA3Kb2T4hzdwmFSs7796lTpxQYGHjZtrfFESoAAIAbiUAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIu83F3A7WTbmNgrfls1AAAofDhCBQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARn5R+E9Ue9aU87H7uLgO3oP0T4txdAgDAAo5QAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLimygio+PV6dOndxdBgAAuA3c0oEqPj5eNptNNptN3t7eqly5soYMGaL09HR3lwYAAODk5e4CrqRt27aaO3euMjMztWrVKvXt21fp6emaMWOGu0sDAACQdIsfoZIku92u0NBQhYWFqXv37urRo4eWLl0qSdq+fbvi4uIUGBioYsWKqVmzZtqzZ0+e/XzxxRe65557VLx4cQUHB6tDhw4ubS9cuKBnnnlGZcuWlcPhUMWKFTV+/Hjn86NHj1aFChVkt9tVrlw5DRgw4IZuNwAAKDxu+SNUl/L19VVmZqb+97//qXnz5rr33nu1fPlyBQYG6r///a+ysrLyXC49PV2DBg1SnTp1lJ6erpdeekl/+ctflJKSIg8PD02ZMkUff/yxFi1apAoVKujgwYM6ePCgJOnDDz/U5MmTtWDBAt155506cuSItm7dmm+NGRkZysjIcE6npaUV7CAAAIBbSqEKVOvXr9f777+vVq1aadq0aQoKCtKCBQvk7e0tSapevXq+yz744IMu07Nnz1aZMmWUmpqq2rVr68CBA6pWrZruuece2Ww2hYeHO9seOHBAoaGhuu++++Tt7a0KFSqoUaNG+a5r/PjxGjNmjMWtBQAAhcUtf8pv2bJlCggIkMPhUHR0tJo3b66pU6cqJSVFzZo1c4apK9mzZ4+6d++uypUrKzAwUJUqVZL0e1iSfr8APiUlRRERERowYIC++uor57IPPfSQzp07p8qVK6tfv37617/+le+RMElKSEjQqVOnnI+cI10AAKBouuUDVYsWLZSSkqJdu3bp/PnzWrJkicqUKSNfX99r6uf+++/X8ePHNWvWLH377bf69ttvJf1+7ZQkNWjQQPv27dPYsWN17tw5de3aVV26dJEkhYWFadeuXZo2bZp8fX311FNPqXnz5srMzMxzXXa7XYGBgS4PAABQdN3ygcrf319Vq1ZVeHi4y9GoyMhIrVq1Kt9Q82fHjx/Xjh07NGLECLVq1Uo1a9bUyZMnc7ULDAzUww8/rFmzZmnhwoX66KOPdOLECUm/X7v1wAMPaMqUKUpOTtbatWv1/fffF9yGAgCAQqtQXUP1Z88884ymTp2qbt26KSEhQUFBQVq3bp0aNWqkiIgIl7YlSpRQcHCw/vnPf6ps2bI6cOCAXnzxRZc2kydPVtmyZVWvXj15eHho8eLFCg0NVfHixZWUlKSLFy+qcePG8vPz0/z58+Xr6+tynRUAALh93fJHqPITHBys5cuX68yZM4qJidFdd92lWbNm5XlNlYeHhxYsWKBNmzapdu3a+tvf/qaJEye6tAkICFBiYqKioqLUsGFD7d+/X5999pk8PDxUvHhxzZo1S02bNlVkZKS++eYbffLJJwoODr5ZmwsAAG5hNmOMcXcRRV1aWpqCgoIUNnCRPOx+7i4Ht6D9E+LcXQIA4BI579+nTp264vXQhfYIFQAAwK2CQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAi7zcXcDtZNuY2Ct+WzUAACh8OEIFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGflH4T1R71pTzsfu4uo0jZPyHO3SUAAMARKgAAAKsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwqEgFKpvNpqVLl0qS9u/fL5vNppSUFLfWBAAAir5CFah++eUXPfHEE6pQoYLsdrtCQ0MVGxurtWvXSpIOHz6sdu3aXVOfH330kRo3bqygoCAVK1ZMd955pwYPHnwjygcAAEWUl7sLuBYPPvigMjMzNW/ePFWuXFlHjx7VN998oxMnTkiSQkNDr6m/r7/+Wt26ddO4ceP0wAMPyGazKTU1Vd98882NKB8AABRRheYI1W+//abVq1crMTFRLVq0UHh4uBo1aqSEhATFxcVJcj3ll2Pnzp1q0qSJHA6H7rzzTiUnJzufW7Zsme655x4NHTpUERERql69ujp16qSpU6c624wePVr16tXTW2+9pbCwMPn5+emhhx7Sb7/9dhO2GgAAFAaFJlAFBAQoICBAS5cuVUZGxlUvN3ToUA0ePFhbtmxRkyZN9MADD+j48eOSfj+itX37dm3btu2yfezevVuLFi3SJ598oi+++EIpKSl6+umn822fkZGhtLQ0lwcAACi6Ck2g8vLyUlJSkubNm6fixYuradOmGjZsmL777rvLLvfMM8/owQcfVM2aNTVjxgwFBQVp9uzZkqRnn31WDRs2VJ06dVSxYkV169ZNc+bMyRXYzp8/r3nz5qlevXpq3ry5pk6dqgULFujIkSN5rnP8+PEKCgpyPsLCwgpmEAAAwC2p0AQq6fdrqA4dOqSPP/5YsbGxSk5OVoMGDZSUlJTvMtHR0c6fvby8FBUVpR07dkiS/P399emnn2r37t0aMWKEAgICNHjwYDVq1Ehnz551LlehQgWVL1/epc/s7Gzt2rUrz3UmJCTo1KlTzsfBgwctbjkAALiVFapAJUkOh0OtW7fWSy+9pDVr1ig+Pl6jRo26pj5sNpvLdJUqVdS3b1+9/fbb2rx5s1JTU7Vw4cIrLn9pPznsdrsCAwNdHgAAoOgqdIHqUrVq1VJ6enq+z69bt875c1ZWljZt2qQaNWrk275ixYry8/Nz6fPAgQM6dOiQc3rt2rXy8PBQ9erVLVYPAACKgkLzsQnHjx/XQw89pN69eysyMlLFihXTxo0b9dprr6ljx475Ljdt2jRVq1ZNNWvW1OTJk3Xy5En17t1b0u938J09e1bt27dXeHi4fvvtN02ZMkWZmZlq3bq1sw+Hw6FevXpp0qRJSktL04ABA9S1a9dr/pgGAABQNBWaQBUQEKDGjRtr8uTJ2rNnjzIzMxUWFqZ+/fpp2LBh+S43YcIEJSYmasuWLapSpYr+/e9/q1SpUpKkmJgYTZs2TT179tTRo0dVokQJ1a9fX1999ZUiIiKcfVStWlWdO3dW+/btdeLECbVv317Tp0+/4dsMAAAKB5sxxri7iFvZ6NGjtXTpUktfYZOWlvb73X4DF8nD7ldwxUH7J8S5uwQAQBGV8/596tSpK14PXeivoQIAAHA3AhUAAIBFBKorGD16tKXTfQAAoOgjUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwyMvdBdxOto2JveK3VQMAgMKHI1QAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFvFJ6TdR7VFfysPu5+4y3G7/hDh3lwAAQIHiCBUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESguoL9+/fLZrMpJSXF3aUAAIBbVKELVPHx8bLZbLLZbPL29lblypU1ZMgQpaenu7s0AABwm/JydwHXo23btpo7d64yMzO1atUq9e3bV+np6ZoxY8Y19WOM0cWLF+XlVSiHAQAA3CIK3REqSbLb7QoNDVVYWJi6d++uHj16aOnSpXr33XcVFRWlYsWKKTQ0VN27d9cvv/ziXC45OVk2m01ffvmloqKiZLfbtWrVKmVnZysxMVFVq1aV3W5XhQoV9Oqrr7qsc+/evWrRooX8/PxUt25drV279mZvNgAAuEUVykB1KV9fX2VmZurChQsaO3astm7dqqVLl2rfvn2Kj4/P1f7555/X+PHjtWPHDkVGRiohIUGJiYkaOXKkUlNT9f777yskJMRlmeHDh2vIkCFKSUlR9erV9cgjjygrK+smbSEAALiVFfpzXevXr9f777+vVq1aqXfv3s75lStX1pQpU9SoUSOdOXNGAQEBzudefvlltW7dWpJ0+vRpvfnmm/rHP/6hXr16SZKqVKmie+65x2U9Q4YMUVxcnCRpzJgxuvPOO7V7927VqFEjV00ZGRnKyMhwTqelpRXcBgMAgFtOoTxCtWzZMgUEBMjhcCg6OlrNmzfX1KlTtWXLFnXs2FHh4eEqVqyY7r33XknSgQMHXJaPiopy/rxjxw5lZGSoVatWl11nZGSk8+eyZctKksvpxD8bP368goKCnI+wsLDr2UwAAFBIFMpA1aJFC6WkpGjXrl06f/68lixZIn9/f7Vp00YBAQF69913tWHDBv3rX/+SJF24cMFleX9/f+fPvr6+V7VOb29v5882m02SlJ2dnWfbhIQEnTp1yvk4ePDgNW0fAAAoXArlKT9/f39VrVrVZd7OnTv166+/asKECc4jQhs3brxiX9WqVZOvr6+++eYb9e3bt0Dqs9vtstvtBdIXAAC49RXKQJWXChUqyMfHR1OnTlX//v21bds2jR079orLORwOvfDCC3r++efl4+Ojpk2b6tixY9q+fbv69OlzEyoHAACFXaE85ZeX0qVLKykpSYsXL1atWrU0YcIETZo06aqWHTlypAYPHqyXXnpJNWvW1MMPP5zv9VEAAACXshljjLuLKOrS0tJ+vzh94CJ52P3cXY7b7Z8Q5+4SAAC4opz371OnTikwMPCybYvMESoAAAB3IVABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEVe7i7gdrJtTKwCAwPdXQYAAChgHKECAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWMQHe95EtUd9KQ+7n7vLuGH2T4hzdwkAALgFR6gAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEVuD1SjR49WvXr1bkjfycnJstls+u233wqsz/3798tmsyklJaXA+gQAAIXbNQWq+Ph42Wy2XI+2bdveqPoAAABueV7XukDbtm01d+5cl3l2u73ACioomZmZ7i4BAADcJq75lJ/dbldoaKjLo0SJEpIkm82mt956Sx06dJCfn59q1qyptWvXavfu3br33nvl7++v6Oho7dmzJ1e/b731lsLCwuTn56eHHnrI5TTdhg0b1Lp1a5UqVUpBQUGKiYnR5s2bXZa32WyaOXOmOnbsKH9/f73yyiu51nHu3DnFxcXp7rvv1okTJyRJc+fOVc2aNeVwOFSjRg1Nnz7dZZn169erfv36cjgcioqK0pYtW651yAAAQBFX4NdQjR07Vj179lRKSopq1Kih7t2764knnlBCQoI2btwoSXrmmWdcltm9e7cWLVqkTz75RF988YVSUlL09NNPO58/ffq0evXqpVWrVmndunWqVq2a2rdvr9OnT7v0M2rUKHXs2FHff/+9evfu7fLcqVOn1KZNG124cEHffPONSpYsqVmzZmn48OF69dVXtWPHDo0bN04jR47UvHnzJEnp6enq0KGDIiIitGnTJo0ePVpDhgy54hhkZGQoLS3N5QEAAIquaz7lt2zZMgUEBLjMe+GFFzRy5EhJ0uOPP66uXbs650dHR2vkyJGKjY2VJD333HN6/PHHXZY/f/685s2bp/Lly0uSpk6dqri4OL3++usKDQ1Vy5YtXdq/9dZbKlGihFauXKkOHTo453fv3t0lSO3bt0+SdPToUT388MOqUqWKPvjgA/n4+Ej6Pfy9/vrr6ty5sySpUqVKSk1N1VtvvaVevXrpvffe08WLFzVnzhz5+fnpzjvv1M8//6wnn3zysmM0fvx4jRkz5ipGEwAAFAXXHKhatGihGTNmuMwrWbKk8+fIyEjnzyEhIZKkOnXquMw7f/680tLSFBgYKEmqUKGCM0xJUnR0tLKzs7Vr1y6Fhobql19+0UsvvaTly5fr6NGjunjxos6ePasDBw641BEVFZVnzffdd58aNmyoRYsWydPTU5J07NgxHTx4UH369FG/fv2cbbOyshQUFCRJ2rFjh+rWrSs/Pz+X2q4kISFBgwYNck6npaUpLCzsissBAIDC6ZoDlb+/v6pWrZrv897e3s6fbTZbvvOys7Pz7SOnTc6/8fHxOnbsmN544w2Fh4fLbrcrOjpaFy5cyFVbXuLi4vTRRx8pNTXVGe5y1j9r1iw1btzYpX1O6DLG5Fvj5djt9lvyQn0AAHBjXHOguhEOHDigQ4cOqVy5cpKktWvXysPDQ9WrV5ckrVq1StOnT1f79u0lSQcPHtSvv/561f1PmDBBAQEBatWqlZKTk1WrVi2FhITojjvu0N69e9WjR488l6tVq5bmz5+vc+fOydfXV5K0bt06K5sKAACKoGsOVBkZGTpy5IhrJ15eKlWq1HUX4XA41KtXL02aNElpaWkaMGCAunbtqtDQUElS1apVNX/+fEVFRSktLU1Dhw51BpyrNWnSJF28eFEtW7ZUcnKyatSoodGjR2vAgAEKDAxUu3btlJGRoY0bN+rkyZMaNGiQunfvruHDh6tPnz4aMWKE9u/fr0mTJl33dgIAgKLpmu/y++KLL1S2bFmXxz333GOpiKpVq6pz585q37692rRpo9q1a7t8fMGcOXN08uRJ1a9fX4899pgGDBigMmXKXPN6Jk+erK5du6ply5b64Ycf1LdvX7399ttKSkpSnTp1FBMTo6SkJFWqVEmSFBAQoE8++USpqamqX7++hg8frsTEREvbCgAAih6bud4LhXDV0tLSFBQUpLCBi+Rh97vyAoXU/glx7i4BAIACk/P+ferUKeeNdPlx+3f5AQAAFHYEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABY5OXuAm4n28bEXvHbqgEAQOHDESoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCIvdxdwOzDGSJLS0tLcXAkAALhaOe/bOe/jl0OgugmOHz8uSQoLC3NzJQAA4FqdPn1aQUFBl21DoLoJSpYsKUk6cODAFV8QXFlaWprCwsJ08OBBBQYGurucQo2xLFiMZ8FhLAsW43l9jDE6ffq0ypUrd8W2BKqbwMPj90vVgoKC2JELUGBgIONZQBjLgsV4FhzGsmAxntfuag+EcFE6AACARQQqAAAAiwhUN4HdbteoUaNkt9vdXUqRwHgWHMayYDGeBYexLFiM541nM1dzLyAAAADyxREqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESguk7Tp09XpUqV5HA4dNddd2nVqlWXbb9y5Urdddddcjgcqly5smbOnJmrzUcffaRatWrJbrerVq1a+te//nWjyr+lFPRYJiUlyWaz5XqcP3/+Rm7GLeNaxvPw4cPq3r27IiIi5OHhoYEDB+bZjn2zYMaSffPqx3PJkiVq3bq1SpcurcDAQEVHR+vLL7/M1Y59s2DG8nbfNwuEwTVbsGCB8fb2NrNmzTKpqanmueeeM/7+/uann37Ks/3evXuNn5+fee6550xqaqqZNWuW8fb2Nh9++KGzzZo1a4ynp6cZN26c2bFjhxk3bpzx8vIy69atu1mb5RY3Yiznzp1rAgMDzeHDh10et4NrHc99+/aZAQMGmHnz5pl69eqZ5557Llcb9s2CG0v2zasfz+eee84kJiaa9evXmx9++MEkJCQYb29vs3nzZmcb9s2CG8vbed8sKASq69CoUSPTv39/l3k1atQwL774Yp7tn3/+eVOjRg2XeU888YS5++67ndNdu3Y1bdu2dWkTGxtrunXrVkBV35puxFjOnTvXBAUFFXithcG1juefxcTE5BkC2Df/YHUs2Tevbzxz1KpVy4wZM8Y5zb75B6tjeTvvmwWFU37X6MKFC9q0aZPatGnjMr9NmzZas2ZNnsusXbs2V/vY2Fht3LhRmZmZl22TX59FwY0aS0k6c+aMwsPDVb58eXXo0EFbtmwp+A24xVzPeF4N9s0/WB1LiX3zz65lPLOzs3X69Gnnl81L7Jt/ZnUspdtz3yxIBKpr9Ouvv+rixYsKCQlxmR8SEqIjR47kucyRI0fybJ+VlaVff/31sm3y67MouFFjWaNGDSUlJenjjz/WBx98IIfDoaZNm+rHH3+8MRtyi7ie8bwa7Jt/sLrd7JvXP56vv/660tPT1bVrV+c89s0/WB3L23XfLEhe7i6gsLLZbC7Txphc867U/tL519pnUVHQY3n33Xfr7rvvdj7ftGlTNWjQQFOnTtWUKVMKquxb1o3Yj9g3f2d1u9k3r288P/jgA40ePVr//ve/VaZMmQLps7Ar6LG83ffNgkCgukalSpWSp6dnrr8Efvnll1x/MeQIDQ3Ns72Xl5eCg4Mv2ya/PouCGzWWl/Lw8FDDhg2L/F9a1zOeV4N98w8Fvd3sm1cez4ULF6pPnz5avHix7rvvPpfn2Df/YHUsL3W77JsFiVN+18jHx0d33XWX/vOf/7jM/89//qMmTZrkuUx0dHSu9l999ZWioqLk7e192Tb59VkU3KixvJQxRikpKSpbtmzBFH6Lup7xvBrsm3+wOpaXYt+8/Hh+8MEHio+P1/vvv6+4uLhcz7Nv/sHqWF7qdtk3C5Q7roQv7HJuWZ09e7ZJTU01AwcONP7+/mb//v3GGGNefPFF89hjjznb59zq/7e//c2kpqaa2bNn57rV/7///a/x9PQ0EyZMMDt27DATJky4rW7/LcixHD16tPniiy/Mnj17zJYtW8zjjz9uvLy8zLfffnvTt+9mu9bxNMaYLVu2mC1btpi77rrLdO/e3WzZssVs377d+Tz7ZsGNJfvm1Y/n+++/b7y8vMy0adNcbuP/7bffnG3YNwtuLG/nfbOgEKiu07Rp00x4eLjx8fExDRo0MCtXrnQ+16tXLxMTE+PSPjk52dSvX9/4+PiYihUrmhkzZuTqc/HixSYiIsJ4e3ubGjVqmI8++uhGb8YtoaDHcuDAgaZChQrGx8fHlC5d2rRp08asWbPmZmzKLeFax1NSrkd4eLhLG/bNghlL9s2rH8+YmJg8x7NXr14ufbJvFsxY3u77ZkGwGfP/r+gFAADAdeEaKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAhUZ8fLw6derk7jLytH//ftlsNqWkpLi7FABuQKACAIsuXLjg7hIAuBmBCkChdO+99+rZZ5/VwIEDVaJECYWEhOif//yn0tPT9fjjj6tYsWKqUqWKPv/8c+cyycnJstls+vTTT1W3bl05HA41btxY33//vUvfH330ke68807Z7XZVrFhRr7/+usvzFStW1CuvvKL4+HgFBQWpX79+qlSpkiSpfv36stlsuvfeeyVJGzZsUOvWrVWqVCkFBQUpJiZGmzdvdunPZrPp7bff1l/+8hf5+fmpWrVq+vjjj13abN++XXFxcQoMDFSxYsXUrFkz7dmzx/n83LlzVbNmTTkcDtWoUUPTp0+3PMYArh6BCkChNW/ePJUqVUrr16/Xs88+qyeffFIPPfSQmjRpos2bNys2NlaPPfaYzp4967Lc0KFDNWnSJG3YsEFlypTRAw88oMzMTEnSpk2b1LVrV3Xr1k3ff/+9Ro8erZEjRyopKcmlj4kTJ6p27dratGmTRo4cqfXr10uSvv76ax0+fFhLliyRJJ0+fVq9evXSqlWrtG7dOlWrVk3t27fX6dOnXfobM2aMunbtqu+++07t27dXjx49dOLECUnS//73PzVv3lwOh0PLly/Xpk2b1Lt3b2VlZUmSZs2apeHDh+vVV1/Vjh07NG7cOI0cOVLz5s0r8DEHkA93fzszAFytXr16mY4dOxpjjImJiTH33HOP87msrCzj7+9vHnvsMee8w4cPG0lm7dq1xhhjVqxYYSSZBQsWONscP37c+Pr6moULFxpjjOnevbtp3bq1y3qHDh1qatWq5ZwODw83nTp1cmmzb98+I8ls2bLlstuQlZVlihUrZj755BPnPElmxIgRzukzZ84Ym81mPv/8c2OMMQkJCaZSpUrmwoULefYZFhZm3n//fZd5Y8eONdHR0ZetBUDB4QgVgEIrMjLS+bOnp6eCg4NVp04d57yQkBBJ0i+//OKyXHR0tPPnkiVLKiIiQjt27JAk7dixQ02bNnVp37RpU/3444+6ePGic15UVNRV1fjLL7+of//+ql69uoKCghQUFKQzZ87owIED+W6Lv7+/ihUr5qw7JSVFzZo1k7e3d67+jx07poMHD6pPnz4KCAhwPl555RWXU4IAbiwvdxcAANfr0oBhs9lc5tlsNklSdnb2FfvKaWuMcf6cwxiTq72/v/9V1RgfH69jx47pjTfeUHh4uOx2u6Kjo3NdyJ7XtuTU7evrm2//OW1mzZqlxo0buzzn6el5VTUCsI5ABeC2s27dOlWoUEGSdPLkSf3www+qUaOGJKlWrVpavXq1S/s1a9aoevXqlw0oPj4+kuRyFEuSVq1apenTp6t9+/aSpIMHD+rXX3+9pnojIyM1b948ZWZm5gpeISEhuuOOO7R371716NHjmvoFUHAIVABuOy+//LKCg4MVEhKi4cOHq1SpUs7Ptxo8eLAaNmyosWPH6uGHH9batWv1j3/844p3zZUpU0a+vr764osvVL58eTkcDgUFBalq1aqaP3++oqKilJaWpqFDh172iFNennnmGU2dOlXdunVTQkKCgoKCtG7dOjVq1EgREREaPXq0BgwYoMDAQLVr104ZGRnauHGjTp48qUGDBl3vMAG4BlxDBeC2M2HCBD333HO66667dPjwYX388cfOI0wNGjTQokWLtGDBAtWuXVsvvfSSXn75ZcXHx1+2Ty8vL02ZMkVvvfWWypUrp44dO0qS5syZo5MnT6p+/fp67LHHNGDAAJUpU+aa6g0ODtby5ct15swZxcTE6K677tKsWbOcR6v69u2rt99+W0lJSapTp45iYmKUlJTk/CgHADeezeR1cQAAFEHJyclq0aKFTp48qeLFi7u7HABFCEeoAAAALCJQAQAAWMQpPwAAAIs4QgUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABg0f8DEw858oGallIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get feature importances and sort features by importance\n",
    "rf_importances = rf_model.feature_importances_\n",
    "indices = np.argsort(rf_importances)  \n",
    "sorted_features = X_train.columns[indices]\n",
    "sorted_importances = rf_importances[indices]\n",
    "\n",
    "# Plot (barh displays bottom to top, so ascending = highest at top)\n",
    "plt.barh(sorted_features, sorted_importances)\n",
    "plt.xlabel('Importance')\n",
    "plt.title('Feature Importances in Random Forest')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0225a6b",
   "metadata": {},
   "source": [
    "Similar to Bagging, `Sex`, `Age`, and `Fare` remain the most important features. However, Random Forest ranks `Age` higher than `Sex`, showing how random feature selection can shift feature importance rankings."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3cffed5",
   "metadata": {},
   "source": [
    "## AdaBoost\n",
    "\n",
    "We implement AdaBoost using scikit-learn's `AdaBoostClassifier`. See [here](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html) for documentation.\n",
    "\n",
    "We set the following parameters:\n",
    "- **`estimator`**: The base model to use. We use `DecisionTreeClassifier(max_depth=1)` to create stumps (trees with only one split).\n",
    "- **`n_estimators`**: The number of boosting iterations (stumps to create). We use 100.\n",
    "- **`random_state`**: Ensures reproducibility."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "be6c3ac5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost Accuracy: 0.8252\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "# Create an AdaBoost classifier\n",
    "adaboost_model = AdaBoostClassifier(\n",
    "    estimator=DecisionTreeClassifier(max_depth=1),\n",
    "    n_estimators=100,\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "# Train the model\n",
    "adaboost_model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_pred_adaboost = adaboost_model.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "adaboost_accuracy = accuracy_score(y_test, y_pred_adaboost)\n",
    "print(f\"AdaBoost Accuracy: {adaboost_accuracy:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64c73f1b",
   "metadata": {},
   "source": [
    "For our dataset, AdaBoost shows the best prediction accuracy so far of 82.52%. Let us visualize the feature importance in this case. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7dc8d46c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAHFCAYAAAA0SmdSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCDklEQVR4nO3dfXzO9f////sxm2PbsRNMjJo5H3JuCLGca+tdSknE9nby7lzeUVoIKSx6601RaUxKTkpCRYX14Y2cLkQqLHu/nSQn2yxms+fvj347vh1tw7x27na9XF6XdryO5+v5eryex3F03D1fr+M4bMYYIwAAAFw3t+IuAAAAoLQjUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABVxEXFyebzZbrMmrUqELZ5/79+zVhwgQlJiYWSv9WJCYmymazafr06cVdynXbvHmzJkyYoHPnzhV3KUUmKipKNWvWLJS+9+7dK5vNJg8PDx0/fvyat8t+LsXFxeV7n/Hx8TlejxUrVlTbtm21YMGCfPdXGCZPnqwVK1YUdxkoIgQq4BrNnz9fW7ZscVmGDx9eKPvav3+/Jk6cWCIDVVmwefNmTZw48YYKVOPGjdMnn3xSKH2/++67kqTMzEy99957hbKPvEyePNn5ely4cKGCg4MVFRWlWbNmFWkdedVGoLpxuBd3AUBp0bhxY4WGhhZ3GZZkZGTIZrPJ3f3GfOlfuHBBnp6exV1GsahTp06h9Juenq4PPvhAzZo102+//aZ58+Zp9OjRhbKv3NSrV0+33Xab83Z4eLi2b9+uDz/8UE899VSR1QEwQwUUkCVLlqhdu3ZyOBzy8fFRz549tXv3bpc2O3bsUL9+/VSzZk15eXmpZs2aeuihh/TLL78428TFxemBBx6QJHXu3Nl5OiP7tEjNmjUVFRWVY/933HGH7rjjDuft7FMiCxcu1MiRI3XzzTfLbrfr559/liR9/fXX6tq1q/z8/OTt7a0OHTpo3bp113Xs2adF169fr2HDhikgIEB+fn4aNGiQ0tLSdOLECfXt21cVKlRQtWrVNGrUKGVkZDi3zz718+qrr+qVV15RjRo15OnpqdDQ0Fxr2rRpk7p27SpfX195e3urffv2+uyzz3Kt6csvv9TgwYN10003ydvbW9HR0Xr22WclSbVq1XKOb3x8vKQ/HscePXqoWrVq8vLyUsOGDfX8888rLS3Npf+oqCj5+Pjo559/Vnh4uHx8fBQUFKSRI0cqPT3dpW16erpeeuklNWzYUJ6engoICFDnzp21efNmZxtjjGbPnq3mzZvLy8tLFStW1P3336/Dhw+79LV7927dddddqlKliux2u6pXr66IiAj997//veJjlNspP5vNpieffFILFy5Uw4YN5e3trWbNmmn16tVX7OvPVqxYodOnT2vo0KGKjIzUjz/+qE2bNuVod+zYMfXt21e+vr7y9/fXgw8+qBMnTuRody2vkStxc3OTj4+PPDw8XNZfvHhR0dHRqlWrlsqXL6+bb75ZTzzxRI5ZyqysLL366qtq0KCB7Ha7qlSpokGDBuUY36s9DjabTWlpaVqwYIHzOfbn1yfKHgIVcI0uX76szMxMlyXb5MmT9dBDD6lRo0ZaunSpFi5cqNTUVHXs2FH79+93tktMTFRISIhef/11rV27VjExMTp+/Lhat26t3377TZIUERGhyZMnS5LefPNN5+mMiIiI66o7OjpaR48e1VtvvaVVq1apSpUqev/999WjRw/5+flpwYIFWrp0qSpVqqSePXted6iSpKFDh8rf31+LFy/W2LFjtWjRIg0bNkwRERFq1qyZPvroI0VGRuq1117L9ZTMG2+8oTVr1uj111/X+++/Lzc3N915553asmWLs80333yjLl26KDk5WbGxsfrwww/l6+urv/3tb1qyZEmOPgcPHiwPDw8tXLhQH330kR577DHnzMXy5cud49uyZUtJ0k8//aTw8HDFxsZqzZo1GjFihJYuXaq//e1vOfrOyMjQ3Xffra5du+rTTz/V4MGDNWPGDMXExDjbZGZm6s4779SkSZN011136ZNPPlFcXJzat2+vo0ePOts98sgjGjFihLp166YVK1Zo9uzZ+v7779W+fXudPHlSkpSWlqbu3bvr5MmTevPNN/XVV1/p9ddfV40aNZSamnpdj9lnn32mN954Qy+99JI+/vhjVapUSffee2+OIJeX2NhY2e12DRgwQIMHD5bNZlNsbKxLmwsXLqhbt2768ssvNWXKFC1btkyBgYF68MEHc/R3La+RP8vKynK+Hk+ePKmpU6dq3759evjhh51tjDHq3bu3pk+froEDB+qzzz7TM888owULFqhLly4uAfixxx7T6NGj1b17d61cuVKTJk3SmjVr1L59e+f+r+Vx2LJli7y8vBQeHu58js2ePfuaxhSllAFwRfPnzzeScl0yMjLM0aNHjbu7u3nqqadctktNTTWBgYGmb9++efadmZlpzp8/bxwOh/n3v//tXL9s2TIjyWzYsCHHNsHBwSYyMjLH+rCwMBMWFua8vWHDBiPJdOrUyaVdWlqaqVSpkvnb3/7msv7y5cumWbNmpk2bNlcYDWOOHDliJJlp06Y512WP0V/HoHfv3kaS+de//uWyvnnz5qZly5Y5+qxevbq5cOGCc31KSoqpVKmS6datm3PdbbfdZqpUqWJSU1Od6zIzM03jxo3NLbfcYrKyslxqGjRoUI5jmDZtmpFkjhw5csVjzcrKMhkZGeabb74xksx3333nvC8yMtJIMkuXLnXZJjw83ISEhDhvv/fee0aSmTt3bp772bJli5FkXnvtNZf1SUlJxsvLyzz33HPGGGN27NhhJJkVK1Zcse7cREZGmuDgYJd1kkzVqlVNSkqKc92JEyeMm5ubmTJlylX7TExMNG5ubqZfv37OdWFhYcbhcLj0OWfOHCPJfPrppy7bDxs2zEgy8+fPz3Mfeb1Gsp/ff13c3NzMmDFjXPpYs2aNkWReffVVl/VLliwxksw777xjjDHmwIEDRpJ5/PHHXdp9++23RpJ54YUXjDHX/jg4HI5cX6som5ihAq7Re++9p+3bt7ss7u7uWrt2rTIzMzVo0CCX2StPT0+FhYU5TyVJ0vnz5zV69GjVrVtX7u7ucnd3l4+Pj9LS0nTgwIFCqbtPnz4utzdv3qwzZ84oMjLSpd6srCz16tVL27dvz3F661rdddddLrcbNmwoSTlm1xo2bJjrKZz77rvP5Rqn7Jmn//u//9Ply5eVlpamb7/9Vvfff798fHyc7cqVK6eBAwfqv//9rw4ePHjF47+aw4cPq3///goMDFS5cuXk4eGhsLAwScrxGNlsthwzV02bNnU5ti+++EKenp4aPHhwnvtcvXq1bDabHn74YZfHJDAwUM2aNXM+h+rWrauKFStq9OjReuutt1xmP69X586d5evr67xdtWpVValS5ZpOsc2fP19ZWVkuxzZ48GClpaW5zBZu2LBBvr6+uvvuu12279+/f44+8/saiYmJcb4ev/rqKz333HOaOnWq87SuJK1fv16Scpwqf+CBB+RwOJyzshs2bMi1XZs2bdSwYUNnu8J4HFD63ZhXpgLXoWHDhrlelJ59OqZ169a5bufm9v/+3dK/f3+tW7dO48aNU+vWreXn5yebzabw8HBduHChUOquVq1arvXef//9eW5z5swZORyOfO+rUqVKLrfLly+f5/qLFy/m2D4wMDDXdZcuXdL58+eVmpoqY0yOY5Kk6tWrS5JOnz7tsj63tnk5f/68OnbsKE9PT7388suqX7++vL29lZSUpPvuuy/HY+Tt7Z3jIne73e5ybKdOnVL16tVdngd/dfLkSRljVLVq1Vzvr127tiTJ399f33zzjV555RW98MILOnv2rKpVq6Zhw4Zp7NixOa4buhYBAQE51tnt9qs+H7OyshQXF6fq1aurVatWzmuRunXrJofDodjYWA0dOlTSH49JbseW2+Od39dI7dq1XV6X3bp109mzZ/Xaa69pyJAhatCggU6fPi13d3fddNNNLtvabDYFBgY6nzPZ/83r+ZUdMgvjcUDpR6ACLKpcubIk6aOPPlJwcHCe7ZKTk7V69WqNHz9ezz//vHN9enq6zpw5c8378/T0zHHRsyT99ttvzlr+zGaz5VrvrFmzXD4d9Wd5vbEXttwuUj5x4oTKly8vHx8fubu7y83NLdfvOjp27Jgk5RiDvx7/laxfv17Hjh1TfHy8c1ZKkqWvV7jpppu0adMmZWVl5RmqKleuLJvNpo0bN8put+e4/8/rmjRposWLF8sYoz179iguLk4vvfSSvLy8XJ5Xhe3rr792BozcQtnWrVu1f/9+NWrUSAEBAdq2bVuONn99vAvqNdK0aVPn+DRo0EABAQHKzMzUqVOnXEKVMUYnTpxw/mMo+ziOHz+uW265xaXPY8eOuTy3SsrjgJKDU36ART179pS7u7sOHTqk0NDQXBfpjzd2Y0yON8x3331Xly9fdlmX3Sa3f5HXrFlTe/bscVn3448/5jjVlZcOHTqoQoUK2r9/f571Zs8sFbXly5e7zO6kpqZq1apV6tixo8qVKyeHw6G2bdtq+fLlLmOTlZWl999/X7fccovq169/1f3kNb7Z4euvj9Hbb7993cd055136uLFi1f88sq77rpLxhj973//y/XxaNKkSY5tbDabmjVrphkzZqhChQratWvXddd4PWJjY+Xm5qYVK1Zow4YNLsvChQslSfPmzZP0x2nF1NRUrVy50qWPRYsWudzOz2vkShISEiRJVapUkSR17dpVkvT++++7tPv444+VlpbmvL9Lly65ttu+fbsOHDjgbPfXmvN6HK5lpg9lBzNUgEU1a9bUSy+9pDFjxujw4cPq1auXKlasqJMnT2rbtm1yOByaOHGi/Pz81KlTJ02bNk2VK1dWzZo19c033yg2NlYVKlRw6bNx48aSpHfeeUe+vr7y9PRUrVq1FBAQoIEDB+rhhx/W448/rj59+uiXX37Rq6++muN0Rl58fHw0a9YsRUZG6syZM7r//vtVpUoVnTp1St99951OnTqlOXPmFPQwXZNy5cqpe/fueuaZZ5SVlaWYmBilpKRo4sSJzjZTpkxR9+7d1blzZ40aNUrly5fX7NmztW/fPn344YfXNCOVHVD+/e9/KzIyUh4eHgoJCVH79u1VsWJFPfrooxo/frw8PDz0wQcf6LvvvrvuY3rooYc0f/58Pfroozp48KA6d+6srKwsffvtt2rYsKH69eunDh066B//+If+/ve/a8eOHerUqZMcDoeOHz+uTZs2qUmTJnrssce0evVqzZ49W71791bt2rVljNHy5ct17tw5de/e/bprzK/Tp0/r008/Vc+ePXXPPffk2mbGjBl67733NGXKFA0aNEgzZszQoEGD9Morr6hevXr6/PPPtXbtWpdt8vMayfbTTz9p69atkv6Y4fr6668VGxur0NBQdezYUZLUvXt39ezZU6NHj1ZKSoo6dOigPXv2aPz48WrRooUGDhwoSQoJCdE//vEPzZo1y/kJ08TERI0bN05BQUH65z//KUnX/Dg0adJE8fHxWrVqlapVqyZfX1+FhIRYGnuUYMV0MTxQamR/Wmz79u1XbLdixQrTuXNn4+fnZ+x2uwkODjb333+/+frrr51t/vvf/5o+ffqYihUrGl9fX9OrVy+zb9++XD+59/rrr5tatWqZcuXKuXwSKisry7z66qumdu3axtPT04SGhpr169fn+Sm/ZcuW5VrvN998YyIiIkylSpWMh4eHufnmm01ERESe7bNd6VN+fx2j8ePHG0nm1KlTLusjIyONw+HI0WdMTIyZOHGiueWWW0z58uVNixYtzNq1a3PUsHHjRtOlSxfjcDiMl5eXue2228yqVatc2lztcYuOjjbVq1c3bm5uLp+o3Lx5s2nXrp3x9vY2N910kxk6dKjZtWtXjk+j/fUY/nrMf3bhwgXz4osvmnr16pny5cubgIAA06VLF7N582aXdvPmzTNt27Z1HledOnXMoEGDzI4dO4wxxvzwww/moYceMnXq1DFeXl7G39/ftGnTxsTFxeV6jH+W16f8nnjiiRxt8/okabbXX3/9qp9ye+utt4wk8/HHHxtj/t9z38fHx/j6+po+ffqYzZs35xjXa32N5PYpP4fDYRo1amTGjx9vkpOTXeq5cOGCGT16tAkODjYeHh6mWrVq5rHHHjNnz551aXf58mUTExNj6tevbzw8PEzlypXNww8/bJKSkpxtrvVxSEhIMB06dDDe3t5GksvrE2WPzRhjijbCAYCrxMRE1apVS9OmTSu030cEgMLENVQAAAAWEagAAAAs4pQfAACARcxQAQAAWESgAgAAsIhABQAAYBFf7FkEsrKydOzYMfn6+ubrZzAAAEDxMcYoNTX1qr/HKRGoisSxY8cUFBRU3GUAAIDrkJSUlOP3Hf+KQFUEfH19Jf3xgPj5+RVzNQAA4FqkpKQoKCjI+T5+JQSqIpB9ms/Pz49ABQBAKXMtl+twUToAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsMi9uAu4kTQev1Zudu/iLgMAgDIlcWpEcZfADBUAAIBVBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsKjMBaqoqCjZbLYcy88//1zcpQEAgDLKvbgLKAy9evXS/PnzXdbddNNN+erj8uXLstlscnMrc5kTAAAUsDKZFux2uwIDA12Wf//732rSpIkcDoeCgoL0+OOP6/z5885t4uLiVKFCBa1evVqNGjWS3W7XL7/8okuXLum5557TzTffLIfDobZt2yo+Pr74Dg4AAJQ4ZTJQ5cbNzU0zZ87Uvn37tGDBAq1fv17PPfecS5vff/9dU6ZM0bvvvqvvv/9eVapU0d///nf95z//0eLFi7Vnzx498MAD6tWrl3766adiOhIAAFDSlMlTfqtXr5aPj4/z9p133qlly5Y5b9eqVUuTJk3SY489ptmzZzvXZ2RkaPbs2WrWrJkk6dChQ/rwww/13//+V9WrV5ckjRo1SmvWrNH8+fM1efLkXPefnp6u9PR05+2UlJQCPT4AAFCylMlA1blzZ82ZM8d52+FwaMOGDZo8ebL279+vlJQUZWZm6uLFi0pLS5PD4ZAklS9fXk2bNnVut2vXLhljVL9+fZf+09PTFRAQkOf+p0yZookTJxbwUQEAgJKqTAYqh8OhunXrOm//8ssvCg8P16OPPqpJkyapUqVK2rRpk4YMGaKMjAxnOy8vL9lsNuftrKwslStXTjt37lS5cuVc9vHnGbC/io6O1jPPPOO8nZKSoqCgoII4NAAAUAKVyUD1Vzt27FBmZqZee+0156f2li5detXtWrRoocuXL+vXX39Vx44dr3l/drtddrv9uusFAAClyw1xUXqdOnWUmZmpWbNm6fDhw1q4cKHeeuutq25Xv359DRgwQIMGDdLy5ct15MgRbd++XTExMfr888+LoHIAAFAa3BCBqnnz5vrXv/6lmJgYNW7cWB988IGmTJlyTdvOnz9fgwYN0siRIxUSEqK7775b3377LafwAACAk80YY4q7iLIuJSVF/v7+ChqxVG527+IuBwCAMiVxakSh9Jv9/p2cnCw/P78rtr0hZqgAAAAKE4EKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABa5F3cBN5J9E3te9deqAQBA6cMMFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFfFN6EWo8fq3c7N7FXQZQqiVOjSjuEgAgB2aoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLbohAtXnzZpUrV069evUq7lIAAEAZdEMEqnnz5umpp57Spk2bdPTo0eIuBwAAlDFlPlClpaVp6dKleuyxx3TXXXcpLi7O5f6VK1eqXr168vLyUufOnbVgwQLZbDadO3fO2Wbz5s3q1KmTvLy8FBQUpOHDhystLa1oDwQAAJRYZT5QLVmyRCEhIQoJCdHDDz+s+fPnyxgjSUpMTNT999+v3r17KyEhQY888ojGjBnjsv3evXvVs2dP3XfffdqzZ4+WLFmiTZs26cknn8xzn+np6UpJSXFZAABA2VXmA1VsbKwefvhhSVKvXr10/vx5rVu3TpL01ltvKSQkRNOmTVNISIj69eunqKgol+2nTZum/v37a8SIEapXr57at2+vmTNn6r333tPFixdz3eeUKVPk7+/vXIKCggr1GAEAQPEq04Hq4MGD2rZtm/r16ydJcnd314MPPqh58+Y572/durXLNm3atHG5vXPnTsXFxcnHx8e59OzZU1lZWTpy5Eiu+42OjlZycrJzSUpKKoSjAwAAJYV7cRdQmGJjY5WZmambb77Zuc4YIw8PD509e1bGGNlsNpdtsk8HZsvKytIjjzyi4cOH5+i/Ro0aue7XbrfLbrcXwBEAAIDSoMwGqszMTL333nt67bXX1KNHD5f7+vTpow8++EANGjTQ559/7nLfjh07XG63bNlS33//verWrVvoNQMAgNKpzAaq1atX6+zZsxoyZIj8/f1d7rv//vsVGxur5cuX61//+pdGjx6tIUOGKCEhwfkpwOyZq9GjR+u2227TE088oWHDhsnhcOjAgQP66quvNGvWrKI+LAAAUAKV2WuoYmNj1a1btxxhSvpjhiohIUFnz57VRx99pOXLl6tp06aaM2eO81N+2afsmjZtqm+++UY//fSTOnbsqBYtWmjcuHGqVq1akR4PAAAouWzmrxcN3eBeeeUVvfXWWwV6IXlKSsofn/YbsVRudu8C6xe4ESVOjSjuEgDcILLfv5OTk+Xn53fFtmX2lN+1mj17tlq3bq2AgAD95z//0bRp0674HVMAAAB/dcMHqp9++kkvv/yyzpw5oxo1amjkyJGKjo4u7rIAAEApcsMHqhkzZmjGjBnFXQYAACjFyuxF6QAAAEWFQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABg0Q3/0zNFad/Enlf9tWoAAFD6MEMFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBHflF6EGo9fKze7d3GXYVni1IjiLgEAgBKFGSoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsKhMBSqbzaYVK1ZIkhITE2Wz2ZSQkFCsNQEAgLKvVAWqX3/9VY888ohq1Kghu92uwMBA9ezZU1u2bJEkHT9+XHfeeWe++vz444/Vtm1b+fv7y9fXV7feeqtGjhxZGOUDAIAyyr24C8iPPn36KCMjQwsWLFDt2rV18uRJrVu3TmfOnJEkBQYG5qu/r7/+Wv369dPkyZN19913y2azaf/+/Vq3bl1hlA8AAMqoUjNDde7cOW3atEkxMTHq3LmzgoOD1aZNG0VHRysiIkKS6ym/bD/88IPat28vT09P3XrrrYqPj3fet3r1at1+++169tlnFRISovr166t3796aNWuWs82ECRPUvHlzvf322woKCpK3t7ceeOABnTt3rgiOGgAAlAalJlD5+PjIx8dHK1asUHp6+jVv9+yzz2rkyJHavXu32rdvr7vvvlunT5+W9MeM1vfff699+/ZdsY+ff/5ZS5cu1apVq7RmzRolJCToiSeesHQ8AACg7Cg1gcrd3V1xcXFasGCBKlSooA4dOuiFF17Qnj17rrjdk08+qT59+qhhw4aaM2eO/P39FRsbK0l66qmn1Lp1azVp0kQ1a9ZUv379NG/evByB7eLFi1qwYIGaN2+uTp06adasWVq8eLFOnDiR6z7T09OVkpLisgAAgLKr1AQq6Y9rqI4dO6aVK1eqZ8+eio+PV8uWLRUXF5fnNu3atXP+7e7urtDQUB04cECS5HA49Nlnn+nnn3/W2LFj5ePjo5EjR6pNmzb6/fffndvVqFFDt9xyi0ufWVlZOnjwYK77nDJlivz9/Z1LUFCQxSMHAAAlWakKVJLk6emp7t2768UXX9TmzZsVFRWl8ePH56sPm83mcrtOnToaOnSo3n33Xe3atUv79+/XkiVLrrr9X/vJFh0dreTkZOeSlJSUr/oAAEDpUuoC1V81atRIaWlped6/detW59+ZmZnauXOnGjRokGf7mjVrytvb26XPo0eP6tixY87bW7ZskZubm+rXr59rH3a7XX5+fi4LAAAou0rN1yacPn1aDzzwgAYPHqymTZvK19dXO3bs0Kuvvqp77rknz+3efPNN1atXTw0bNtSMGTN09uxZDR48WNIfn+D7/fffFR4eruDgYJ07d04zZ85URkaGunfv7uzD09NTkZGRmj59ulJSUjR8+HD17ds331/TAAAAyqZSE6h8fHzUtm1bzZgxQ4cOHVJGRoaCgoI0bNgwvfDCC3luN3XqVMXExGj37t2qU6eOPv30U1WuXFmSFBYWpjfffFODBg3SyZMnVbFiRbVo0UJffvmlQkJCnH3UrVtX9913n8LDw3XmzBmFh4dr9uzZhX7MAACgdLAZY0xxF1GSTZgwQStWrLD0EzYpKSl/XJw+Yqnc7N4FV1wxSZwaUdwlAABQ6LLfv5OTk696+U6pv4YKAACguBGoAAAALCJQXcWECRMsne4DAABlH4EKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEXuxV3AjWTfxJ5X/bVqAABQ+jBDBQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGAR35RehBqPXys3u3dxl2FZ4tSI4i4BAIAShRkqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGBRmQ1UUVFR6t27d3GXAQAAbgAlOlBFRUXJZrPJZrPJw8NDtWvX1qhRo5SWllbcpQEAADi5F3cBV9OrVy/Nnz9fGRkZ2rhxo4YOHaq0tDTNmTOnuEsDAACQVMJnqCTJbrcrMDBQQUFB6t+/vwYMGKAVK1ZIkr7//ntFRETIz89Pvr6+6tixow4dOpRrP2vWrNHtt9+uChUqKCAgQHfddZdL20uXLunJJ59UtWrV5OnpqZo1a2rKlCnO+ydMmKAaNWrIbrerevXqGj58eKEeNwAAKD1K/AzVX3l5eSkjI0P/+9//1KlTJ91xxx1av369/Pz89J///EeZmZm5bpeWlqZnnnlGTZo0UVpaml588UXde++9SkhIkJubm2bOnKmVK1dq6dKlqlGjhpKSkpSUlCRJ+uijjzRjxgwtXrxYt956q06cOKHvvvsuzxrT09OVnp7uvJ2SklKwgwAAAEqUUhWotm3bpkWLFqlr165688035e/vr8WLF8vDw0OSVL9+/Ty37dOnj8vt2NhYValSRfv371fjxo119OhR1atXT7fffrtsNpuCg4OdbY8eParAwEB169ZNHh4eqlGjhtq0aZPnvqZMmaKJEydaPFoAAFBalPhTfqtXr5aPj488PT3Vrl07derUSbNmzVJCQoI6duzoDFNXc+jQIfXv31+1a9eWn5+fatWqJemPsCT9cQF8QkKCQkJCNHz4cH355ZfObR944AFduHBBtWvX1rBhw/TJJ5/kORMmSdHR0UpOTnYu2TNdAACgbCrxgapz585KSEjQwYMHdfHiRS1fvlxVqlSRl5dXvvr529/+ptOnT2vu3Ln69ttv9e2330r649opSWrZsqWOHDmiSZMm6cKFC+rbt6/uv/9+SVJQUJAOHjyoN998U15eXnr88cfVqVMnZWRk5Lovu90uPz8/lwUAAJRdJT5QORwO1a1bV8HBwS6zUU2bNtXGjRvzDDV/dvr0aR04cEBjx45V165d1bBhQ509ezZHOz8/Pz344IOaO3eulixZoo8//lhnzpyR9Me1W3fffbdmzpyp+Ph4bdmyRXv37i24AwUAAKVWqbqG6s+efPJJzZo1S/369VN0dLT8/f21detWtWnTRiEhIS5tK1asqICAAL3zzjuqVq2ajh49queff96lzYwZM1StWjU1b95cbm5uWrZsmQIDA1WhQgXFxcXp8uXLatu2rby9vbVw4UJ5eXm5XGcFAABuXCV+hiovAQEBWr9+vc6fP6+wsDC1atVKc+fOzfWaKjc3Ny1evFg7d+5U48aN9c9//lPTpk1zaePj46OYmBiFhoaqdevWSkxM1Oeffy43NzdVqFBBc+fOVYcOHdS0aVOtW7dOq1atUkBAQFEdLgAAKMFsxhhT3EWUdSkpKfL391fQiKVys3sXdzmWJU6NKO4SAAAodNnv38nJyVe9HrrUzlABAACUFAQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFjkXtwF3Ej2Tex51V+rBgAApQ8zVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAW8U3pRajx+LVys3sXaw2JUyOKdf8AAJRFzFABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAItuiED166+/6pFHHlGNGjVkt9sVGBionj17asuWLcVdGgAAKAPci7uAotCnTx9lZGRowYIFql27tk6ePKl169bpzJkzxV0aAAAoA8r8DNW5c+e0adMmxcTEqHPnzgoODlabNm0UHR2tiIgISVJycrL+8Y9/qEqVKvLz81OXLl303XffSZJOnTqlwMBATZ482dnnt99+q/Lly+vLL78slmMCAAAlS5kPVD4+PvLx8dGKFSuUnp6e435jjCIiInTixAl9/vnn2rlzp1q2bKmuXbvqzJkzuummmzRv3jxNmDBBO3bs0Pnz5/Xwww/r8ccfV48ePXLdZ3p6ulJSUlwWAABQdpX5QOXu7q64uDgtWLBAFSpUUIcOHfTCCy9oz549kqQNGzZo7969WrZsmUJDQ1WvXj1Nnz5dFSpU0EcffSRJCg8P17BhwzRgwAA9+uij8vT01NSpU/Pc55QpU+Tv7+9cgoKCiuRYAQBA8SjzgUr64xqqY8eOaeXKlerZs6fi4+PVsmVLxcXFaefOnTp//rwCAgKcs1k+Pj46cuSIDh065Oxj+vTpyszM1NKlS/XBBx/I09Mzz/1FR0crOTnZuSQlJRXFYQIAgGJyQ1yULkmenp7q3r27unfvrhdffFFDhw7V+PHj9fjjj6tatWqKj4/PsU2FChWcfx8+fFjHjh1TVlaWfvnlFzVt2jTPfdntdtnt9kI4CgAAUBLdMIHqrxo1aqQVK1aoZcuWOnHihNzd3VWzZs1c2166dEkDBgzQgw8+qAYNGmjIkCHau3evqlatWrRFAwCAEqnMn/I7ffq0unTpovfff1979uzRkSNHtGzZMr366qu655571K1bN7Vr1069e/fW2rVrlZiYqM2bN2vs2LHasWOHJGnMmDFKTk7WzJkz9dxzz6lhw4YaMmRIMR8ZAAAoKcr8DJWPj4/atm2rGTNm6NChQ8rIyFBQUJCGDRumF154QTabTZ9//rnGjBmjwYMHO78moVOnTqpatari4+P1+uuva8OGDfLz85MkLVy4UE2bNtWcOXP02GOPFfMRAgCA4mYzxpjiLqKsS0lJ+ePTfiOWys3uXay1JE6NKNb9AwBQWmS/fycnJzsnVfJS5k/5AQAAFDYCFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsci/uAm4k+yb2vOqvVQMAgNKHGSoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAi/im9CLUePxaudm9r3v7xKkRBVgNAAAoKMxQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBKqrSExMlM1mU0JCQnGXAgAASqhSF6iioqJks9lks9nk4eGh2rVra9SoUUpLSyvu0gAAwA3KvbgLuB69evXS/PnzlZGRoY0bN2ro0KFKS0vTnDlz8tWPMUaXL1+Wu3upHAYAAFBClLoZKkmy2+0KDAxUUFCQ+vfvrwEDBmjFihV6//33FRoaKl9fXwUGBqp///769ddfndvFx8fLZrNp7dq1Cg0Nld1u18aNG5WVlaWYmBjVrVtXdrtdNWrU0CuvvOKyz8OHD6tz587y9vZWs2bNtGXLlqI+bAAAUEKVykD1V15eXsrIyNClS5c0adIkfffdd1qxYoWOHDmiqKioHO2fe+45TZkyRQcOHFDTpk0VHR2tmJgYjRs3Tvv379eiRYtUtWpVl23GjBmjUaNGKSEhQfXr19dDDz2kzMzMIjpCAABQkpX6c13btm3TokWL1LVrVw0ePNi5vnbt2po5c6batGmj8+fPy8fHx3nfSy+9pO7du0uSUlNT9e9//1tvvPGGIiMjJUl16tTR7bff7rKfUaNGKSIiQpI0ceJE3Xrrrfr555/VoEGDHDWlp6crPT3deTslJaXgDhgAAJQ4pXKGavXq1fLx8ZGnp6fatWunTp06adasWdq9e7fuueceBQcHy9fXV3fccYck6ejRoy7bh4aGOv8+cOCA0tPT1bVr1yvus2nTps6/q1WrJkkupxP/bMqUKfL393cuQUFB13OYAACglCiVgapz585KSEjQwYMHdfHiRS1fvlwOh0M9evSQj4+P3n//fW3fvl2ffPKJJOnSpUsu2zscDuffXl5e17RPDw8P5982m02SlJWVlWvb6OhoJScnO5ekpKR8HR8AAChdSuUpP4fDobp167qs++GHH/Tbb79p6tSpzhmhHTt2XLWvevXqycvLS+vWrdPQoUMLpD673S673V4gfQEAgJKvVAaq3NSoUUPly5fXrFmz9Oijj2rfvn2aNGnSVbfz9PTU6NGj9dxzz6l8+fLq0KGDTp06pe+//15DhgwpgsoBAEBpVypP+eXmpptuUlxcnJYtW6ZGjRpp6tSpmj59+jVtO27cOI0cOVIvvviiGjZsqAcffDDP66MAAAD+ymaMMcVdRFmXkpLyx8XpI5bKze593f0kTo0owKoAAMCVZL9/Jycny8/P74pty8wMFQAAQHEhUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARe7FXcCNZN/EnvLz8yvuMgAAQAFjhgoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEV/sWYQaj18rN7t3nvcnTo0owmoAAEBBYYYKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhU7IFqwoQJat68eaH0HR8fL5vNpnPnzhVYn4mJibLZbEpISCiwPgEAQOmWr0AVFRUlm82WY+nVq1dh1QcAAFDiued3g169emn+/Pku6+x2e4EVVFAyMjKKuwQAAHCDyPcpP7vdrsDAQJelYsWKkiSbzaa3335bd911l7y9vdWwYUNt2bJFP//8s+644w45HA61a9dOhw4dytHv22+/raCgIHl7e+uBBx5wOU23fft2de/eXZUrV5a/v7/CwsK0a9cul+1tNpveeust3XPPPXI4HHr55Zdz7OPChQuKiIjQbbfdpjNnzkiS5s+fr4YNG8rT01MNGjTQ7NmzXbbZtm2bWrRoIU9PT4WGhmr37t35HTIAAFDGFfg1VJMmTdKgQYOUkJCgBg0aqH///nrkkUcUHR2tHTt2SJKefPJJl21+/vlnLV26VKtWrdKaNWuUkJCgJ554wnl/amqqIiMjtXHjRm3dulX16tVTeHi4UlNTXfoZP3687rnnHu3du1eDBw92uS85OVk9evTQpUuXtG7dOlWqVElz587VmDFj9Morr+jAgQOaPHmyxo0bpwULFkiS0tLSdNdddykkJEQ7d+7UhAkTNGrUqKuOQXp6ulJSUlwWAABQhpl8iIyMNOXKlTMOh8Nleemll4wxxkgyY8eOdbbfsmWLkWRiY2Od6z788EPj6enpvD1+/HhTrlw5k5SU5Fz3xRdfGDc3N3P8+PFc68jMzDS+vr5m1apVznWSzIgRI1zabdiwwUgyP/zwg2nWrJm57777THp6uvP+oKAgs2jRIpdtJk2aZNq1a2eMMebtt982lSpVMmlpac7758yZYySZ3bt35zlO48ePN5JyLEEjlprg0avzXAAAQMmRnJxsJJnk5OSrts33NVSdO3fWnDlzXNZVqlTJ+XfTpk2df1etWlWS1KRJE5d1Fy9eVEpKivz8/CRJNWrU0C233OJs065dO2VlZengwYMKDAzUr7/+qhdffFHr16/XyZMndfnyZf3+++86evSoSx2hoaG51tytWze1bt1aS5cuVbly5SRJp06dUlJSkoYMGaJhw4Y522ZmZsrf31+SdODAATVr1kze3t4utV1NdHS0nnnmGeftlJQUBQUFXXU7AABQOuU7UDkcDtWtWzfP+z08PJx/22y2PNdlZWXl2Ud2m+z/RkVF6dSpU3r99dcVHBwsu92udu3a6dKlSzlqy01ERIQ+/vhj7d+/3xnusvc/d+5ctW3b1qV9dugyxuRZ45XY7fYSeaE+AAAoHPkOVIXh6NGjOnbsmKpXry5J2rJli9zc3FS/fn1J0saNGzV79myFh4dLkpKSkvTbb79dc/9Tp06Vj4+Punbtqvj4eDVq1EhVq1bVzTffrMOHD2vAgAG5bteoUSMtXLhQFy5ckJeXlyRp69atVg4VAACUQfkOVOnp6Tpx4oRrJ+7uqly58nUX4enpqcjISE2fPl0pKSkaPny4+vbtq8DAQElS3bp1tXDhQoWGhiolJUXPPvusM+Bcq+nTp+vy5cvq0qWL4uPj1aBBA02YMEHDhw+Xn5+f7rzzTqWnp2vHjh06e/asnnnmGfXv319jxozRkCFDNHbsWCUmJmr69OnXfZwAAKBsyven/NasWaNq1aq5LLfffrulIurWrav77rtP4eHh6tGjhxo3buzy9QXz5s3T2bNn1aJFCw0cOFDDhw9XlSpV8r2fGTNmqG/fvurSpYt+/PFHDR06VO+++67i4uLUpEkThYWFKS4uTrVq1ZIk+fj4aNWqVdq/f79atGihMWPGKCYmxtKxAgCAssdmrvdCIVyzlJQU+fv7K2jEUrnZvfNslzg1ogirAgAAV5L9/p2cnOz8IF1eiv23/AAAAEo7AhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABYRKACAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALHIv7gJuJPsm9rzqr1UDAIDShxkqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAgAAsIhABQAAYBGBCgAAwCICFQAAgEUEKgAAAIsIVAAAABYRqAAAACwiUAEAAFhEoAIAALCIQAUAAGARgQoAAMAi9+Iu4EZgjJEkpaSkFHMlAADgWmW/b2e/j18JgaoInD59WpIUFBRUzJUAAID8Sk1Nlb+//xXbEKiKQKVKlSRJR48eveoDgmuTkpKioKAgJSUlyc/Pr7jLKfUYz4LFeBY8xrRgMZ7Xxhij1NRUVa9e/aptCVRFwM3tj0vV/P39eeIWMD8/P8a0ADGeBYvxLHiMacFiPK/uWidCuCgdAADAIgIVAACARQSqImC32zV+/HjZ7fbiLqXMYEwLFuNZsBjPgseYFizGs+DZzLV8FhAAAAB5YoYKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoCsjs2bNVq1YteXp6qlWrVtq4ceMV23/zzTdq1aqVPD09Vbt2bb311ltFVGnpkJ/xPH78uPr376+QkBC5ublpxIgRRVdoKZKfMV2+fLm6d++um266SX5+fmrXrp3Wrl1bhNWWfPkZz02bNqlDhw4KCAiQl5eXGjRooBkzZhRhtSVffv8fmu0///mP3N3d1bx588ItsBTKz5jGx8fLZrPlWH744YcirLiUM7Bs8eLFxsPDw8ydO9fs37/fPP3008bhcJhffvkl1/aHDx823t7e5umnnzb79+83c+fONR4eHuajjz4q4spLpvyO55EjR8zw4cPNggULTPPmzc3TTz9dtAWXAvkd06efftrExMSYbdu2mR9//NFER0cbDw8Ps2vXriKuvGTK73ju2rXLLFq0yOzbt88cOXLELFy40Hh7e5u33367iCsvmfI7ntnOnTtnateubXr06GGaNWtWNMWWEvkd0w0bNhhJ5uDBg+b48ePOJTMzs4grL70IVAWgTZs25tFHH3VZ16BBA/P888/n2v65554zDRo0cFn3yCOPmNtuu63QaixN8juefxYWFkagyoWVMc3WqFEjM3HixIIurVQqiPG89957zcMPP1zQpZVK1zueDz74oBk7dqwZP348geov8jum2YHq7NmzRVBd2cQpP4suXbqknTt3qkePHi7re/Tooc2bN+e6zZYtW3K079mzp3bs2KGMjIxCq7U0uJ7xxJUVxJhmZWUpNTXV+UPfN7KCGM/du3dr8+bNCgsLK4wSS5XrHc/58+fr0KFDGj9+fGGXWOpYeY62aNFC1apVU9euXbVhw4bCLLPM4ceRLfrtt990+fJlVa1a1WV91apVdeLEiVy3OXHiRK7tMzMz9dtvv6latWqFVm9Jdz3jiSsriDF97bXXlJaWpr59+xZGiaWKlfG85ZZbdOrUKWVmZmrChAkaOnRoYZZaKlzPeP700096/vnntXHjRrm78zb2V9czptWqVdM777yjVq1aKT09XQsXLlTXrl0VHx+vTp06FUXZpR7PxAJis9lcbhtjcqy7Wvvc1t+o8jueuLrrHdMPP/xQEyZM0KeffqoqVaoUVnmlzvWM58aNG3X+/Hlt3bpVzz//vOrWrauHHnqoMMssNa51PC9fvqz+/ftr4sSJql+/flGVVyrl5zkaEhKikJAQ5+127dopKSlJ06dPJ1BdIwKVRZUrV1a5cuVypP5ff/01x78OsgUGBuba3t3dXQEBAYVWa2lwPeOJK7MypkuWLNGQIUO0bNkydevWrTDLLDWsjGetWrUkSU2aNNHJkyc1YcKEGz5Q5Xc8U1NTtWPHDu3evVtPPvmkpD9OSRtj5O7uri+//FJdunQpktpLqoL6/+htt92m999/v6DLK7O4hsqi8uXLq1WrVvrqq69c1n/11Vdq3759rtu0a9cuR/svv/xSoaGh8vDwKLRaS4PrGU9c2fWO6YcffqioqCgtWrRIERERhV1mqVFQz1FjjNLT0wu6vFInv+Pp5+envXv3KiEhwbk8+uijCgkJUUJCgtq2bVtUpZdYBfUc3b179w19CUq+Fdvl8GVI9sdTY2Njzf79+82IESOMw+EwiYmJxhhjnn/+eTNw4EBn++yvTfjnP/9p9u/fb2JjY/nahD/J73gaY8zu3bvN7t27TatWrUz//v3N7t27zffff18c5ZdI+R3TRYsWGXd3d/Pmm2+6fIT63LlzxXUIJUp+x/ONN94wK1euND/++KP58ccfzbx584yfn58ZM2ZMcR1CiXI9r/k/41N+OeV3TGfMmGE++eQT8+OPP5p9+/aZ559/3kgyH3/8cXEdQqlDoCogb775pgkODjbly5c3LVu2NN98843zvsjISBMWFubSPj4+3rRo0cKUL1/e1KxZ08yZM6eIKy7Z8jueknIswcHBRVt0CZefMQ0LC8t1TCMjI4u+8BIqP+M5c+ZMc+uttxpvb2/j5+dnWrRoYWbPnm0uX75cDJWXTPl9zf8ZgSp3+RnTmJgYU6dOHePp6WkqVqxobr/9dvPZZ58VQ9Wll82Y//9qaAAAAFwXrqECAACwiEAFAABgEYEKAADAIgIVAACARQQqAAAAiwhUAAAAFhGoAAAALCJQAQAAWESgAlBqREVFqXfv3sVdRq4SExNls9mUkJBQ3KUAKAYEKgCw6NKlS8VdAoBiRqACUCrdcccdeuqppzRixAhVrFhRVatW1TvvvKO0tDT9/e9/l6+vr+rUqaMvvvjCuU18fLxsNps+++wzNWvWTJ6enmrbtq327t3r0vfHH3+sW2+9VXa7XTVr1tRrr73mcn/NmjX18ssvKyoqSv7+/ho2bJhq1aolSWrRooVsNpvuuOMOSdL27dvVvXt3Va5cWf7+/goLC9OuXbtc+rPZbHr33Xd17733ytvbW/Xq1dPKlStd2nz//feKiIiQn5+ffH191bFjRx06dMh5//z589WwYUN5enqqQYMGmj17tuUxBnDtCFQASq0FCxaocuXK2rZtm5566ik99thjeuCBB9S+fXvt2rVLPXv21MCBA/X777+7bPfss89q+vTp2r59u6pUqaK7775bGRkZkqSdO3eqb9++6tevn/bu3asJEyZo3LhxiouLc+lj2rRpaty4sXbu3Klx48Zp27ZtkqSvv/5ax48f1/LlyyVJqampioyM1MaNG7V161bVq1dP4eHhSk1Ndelv4sSJ6tu3r/bs2aPw8HANGDBAZ86ckST973//U6dOneTp6an169dr586dGjx4sDIzMyVJc+fO1ZgxY/TKK6/owIEDmjx5ssaNG6cFCxYU+JgDyENx/zozAFyryMhIc8899xhjjAkLCzO33367877MzEzjcDjMwIEDneuOHz9uJJktW7YYY4zZsGGDkWQWL17sbHP69Gnj5eVllixZYowxpn///qZ79+4u+3322WdNo0aNnLeDg4NN7969XdocOXLESDK7d+++4jFkZmYaX19fs2rVKuc6SWbs2LHO2+fPnzc2m8188cUXxhhjoqOjTa1atcylS5dy7TMoKMgsWrTIZd2kSZNMu3btrlgLgILDDBWAUqtp06bOv8uVK6eAgAA1adLEua5q1aqSpF9//dVlu3bt2jn/rlSpkkJCQnTgwAFJ0oEDB9ShQweX9h06dNBPP/2ky5cvO9eFhoZeU42//vqrHn30UdWvX1/+/v7y9/fX+fPndfTo0TyPxeFwyNfX11l3QkKCOnbsKA8Pjxz9nzp1SklJSRoyZIh8fHycy8svv+xyShBA4XIv7gIA4Hr9NWDYbDaXdTabTZKUlZV11b6y2xpjnH9nM8bkaO9wOK6pxqioKJ06dUqvv/66goODZbfb1a5duxwXsud2LNl1e3l55dl/dpu5c+eqbdu2LveVK1fummoEYB2BCsANZ+vWrapRo4Yk6ezZs/rxxx/VoEEDSVKjRo20adMml/abN29W/fr1rxhQypcvL0kus1iStHHjRs2ePVvh4eGSpKSkJP3222/5qrdp06ZasGCBMjIycgSvqlWr6uabb9bhw4c1YMCAfPULoOAQqADccF566SUFBASoatWqGjNmjCpXruz8fquRI0eqdevWmjRpkh588EFt2bJFb7zxxlU/NVelShV5eXlpzZo1uuWWW+Tp6Sl/f3/VrVtXCxcuVGhoqFJSUvTss89eccYpN08++aRmzZqlfv36KTo6Wv7+/tq6davatGmjkJAQTZgwQcOHD5efn5/uvPNOpaena8eOHTp79qyeeeaZ6x0mAPnANVQAbjhTp07V008/rVatWun48eNauXKlc4apZcuWWrp0qRYvXqzGjRvrxRdf1EsvvaSoqKgr9unu7q6ZM2fq7bffVvXq1XXPPfdIkubNm6ezZ8+qRYsWGjhwoIYPH64qVarkq96AgACtX79e58+fV1hYmFq1aqW5c+c6Z6uGDh2qd999V3FxcWrSpInCwsIUFxfn/CoHAIXPZnK7OAAAyqD4+Hh17txZZ8+eVYUKFYq7HABlCDNUAAAAFhGoAAAALOKUHwAAgEXMUAEAAFhEoAIAALCIQAUAAGARgQoAAMAiAhUAAIBFBCoAAACLCFQAAAAWEagAAAAsIlABAABY9P8BidpPEjdF6+oAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get feature importances and sort features by importance\n",
    "adaboost_importances = adaboost_model.feature_importances_\n",
    "indices = np.argsort(adaboost_importances)\n",
    "sorted_features = X_train.columns[indices]\n",
    "sorted_importances = adaboost_importances[indices]\n",
    "\n",
    "# Plot\n",
    "plt.barh(sorted_features, sorted_importances)\n",
    "plt.xlabel('Importance')\n",
    "plt.title('Feature Importances in AdaBoost')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03991b26",
   "metadata": {},
   "source": [
    "The feature importance pattern in AdaBoost differs significantly from the other methods. `Fare` is by far the most important feature, with `Age` being moderately important followed by `SibSp`. All other features, including `Sex` which was highly important in Bagging and Random Forests, show much lower importance in AdaBoost. \n",
    "\n",
    "This shows an important characteristic of AdaBoost with stumps: since each stump makes only a single split, the sequential learning process can emphasize different features than methods using deeper trees. This is not an error, but rather reflects how different ensemble approaches can prioritize features differently even on the same data."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef753331",
   "metadata": {},
   "source": [
    "## Hyperparameter Tuning with Grid Search\n",
    "\n",
    "So far, we have manually selected hyperparameters for our models. For example, in Random Forest, we chose `n_estimators=100` and used the default value for `max_features` (which is 'sqrt'). While these are reasonable choices, they may not be optimal. Selecting the right hyperparameters is crucial for achieving the best model performance, but manually testing different combinations can be time-consuming and inefficient.\n",
    "\n",
    "**Grid search** is a systematic method for hyperparameter tuning that automates this process. It evaluates a predefined set of hyperparameter combinations to find the configuration that produces the best performance.\n",
    "\n",
    "Think of grid search as exploring a grid where each axis represents a hyperparameter, and each point on the grid represents a specific combination of hyperparameter values. Grid search exhaustively tests each combination to find the best one.\n",
    "\n",
    "Grid search involves three main components:\n",
    "\n",
    "- **Hyperparameter Space:** The range of values to explore for each hyperparameter. For example, testing `n_estimators` values of 50, 100, and 200.\n",
    "- **Scoring Metric:** The performance metric used to evaluate each combination, such as accuracy or F1-score.\n",
    "- **Cross-Validation:** Recall that cross-validation splits the training data into multiple folds to evaluate performance more reliably, ensuring the results are not due to a particular split of the data.\n",
    "\n",
    "Let us apply grid search to tune our Random Forest model. Note that Random Forest has many hyperparameters, and we have been using their default values so far. Here, we will tune four key hyperparameters:\n",
    "\n",
    "- **`n_estimators`**: The number of trees in the forest.\n",
    "- **`max_depth`**: The maximum depth of each tree. \n",
    "- **`max_features`**: The number of features to consider when looking for the best split. Recall that the default for classification is `'sqrt'`.\n",
    "- **`min_samples_split`**: The minimum number of samples required to split an internal node.\n",
    "\n",
    "Please see the [documentation](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html) for more hyperparameters and their default values. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "42151961",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters: {'max_depth': None, 'max_features': 'sqrt', 'min_samples_split': 10, 'n_estimators': 50}\n",
      "Random Forest Accuracy with Grid Search: 0.8112\n"
     ]
    }
   ],
   "source": [
    "# Import necessary libraries\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# Create a Random Forest Classifier\n",
    "random_forest_model = RandomForestClassifier(random_state=42)\n",
    "\n",
    "# Define the parameter grid for grid search\n",
    "param_grid = {\n",
    "    'n_estimators': [50, 100, 150, 200],\n",
    "    'max_depth': [None, 20, 30],  # Include None\n",
    "    'max_features': ['sqrt', 'log2', None],\n",
    "    'min_samples_split': [2, 5, 10]\n",
    "}\n",
    "\n",
    "# Perform grid search with cross-validation\n",
    "grid_search = GridSearchCV(random_forest_model, param_grid, cv=10, scoring='accuracy')\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "# Get the best parameters from grid search\n",
    "best_params = grid_search.best_params_\n",
    "print(f\"Best Parameters: {best_params}\")\n",
    "\n",
    "# Get the best model \n",
    "best_random_forest_model = grid_search.best_estimator_\n",
    "\n",
    "# Make predictions on the test data\n",
    "predictions = best_random_forest_model.predict(X_test)\n",
    "\n",
    "# Evaluate the model accuracy\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print(f\"Random Forest Accuracy with Grid Search: {accuracy:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c5213c0",
   "metadata": {},
   "source": [
    "In the above grid search, we tested $4 \\times 3 \\times 3 \\times 3 = 108$ different combinations of hyperparameter values using cross-validation. \n",
    "\n",
    "Grid search identified the best combination as: unlimited tree depth (`max_depth=None`), square root of features at each split (`max_features='sqrt'`), minimum of 10 samples required to split a node (`min_samples_split=10`), and 50 trees in the forest (`n_estimators=50`). This configuration achieves an accuracy of 81.12% which is the same as our original Random Forest model. However, there is an important advantage: the tuned model uses only 50 trees instead of 100, meaning it trains approximately twice as fast while maintaining the same predictive performance. This illustrates how hyperparameter tuning can find more efficient configurations without sacrificing performance."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecbd29b5",
   "metadata": {},
   "source": [
    "```{raw} html\n",
    "<!-- NEW_TERMS_START -->\n",
    "<div style=\"border-left:6px solid #800000; background: rgba(255,255,255,0.06) !important; box-shadow:none !important; padding:1rem; border-radius:10px; margin:1rem 0;\">\n",
    "  <div style=\"display:flex; align-items:center; gap:.6rem; margin-bottom:.6rem;\">\n",
    "    <span style=\"display:inline-block; font-weight:700; padding:.25rem .6rem; border:1px solid #800000; border-radius:.5rem; background:rgba(128,0,0,.25); color:inherit;\">\n",
    "      New in This Chapter\n",
    "    </span>\n",
    "  </div>\n",
    "\n",
    "  <div style=\"display:grid; grid-template-columns:repeat(auto-fit,minmax(240px,1fr)); gap:1.25rem; align-items:start;\">\n",
    "    \n",
    "        <div>\n",
    "          <h4 style=\"margin:.25rem 0 .4rem; font-size:1rem; font-weight:700;\">\n",
    "            <span style=\"border-bottom:2px solid rgba(128,0,0,.55); padding-bottom:2px;\">Terms</span>\n",
    "          </h4>\n",
    "          <ul style=\"margin:0; padding-left:1.2rem;\">\n",
    "            <li><a href=\"../../glossary.html#adaboost\" style=\"color:inherit; text-decoration:underline;\">AdaBoost</a></li><li><a href=\"../../glossary.html#bagging\" style=\"color:inherit; text-decoration:underline;\">Bagging</a></li><li><a href=\"../../glossary.html#boosting\" style=\"color:inherit; text-decoration:underline;\">Boosting</a></li><li><a href=\"../../glossary.html#branches\" style=\"color:inherit; text-decoration:underline;\">Branches</a></li><li><a href=\"../../glossary.html#decision-tree\" style=\"color:inherit; text-decoration:underline;\">Decision Tree</a></li><li><a href=\"../../glossary.html#ensemble-methods\" style=\"color:inherit; text-decoration:underline;\">Ensemble Methods</a></li><li><a href=\"../../glossary.html#gini-impurity\" style=\"color:inherit; text-decoration:underline;\">Gini impurity</a></li><li><a href=\"../../glossary.html#gradient-boosting\" style=\"color:inherit; text-decoration:underline;\">Gradient Boosting</a></li><li><a href=\"../../glossary.html#internal-nodes\" style=\"color:inherit; text-decoration:underline;\">Internal Nodes</a></li><li><a href=\"../../glossary.html#leaf-nodes-or-leaves\" style=\"color:inherit; text-decoration:underline;\">Leaf Nodes or Leaves</a></li><li><a href=\"../../glossary.html#post-pruning\" style=\"color:inherit; text-decoration:underline;\">Post-pruning</a></li><li><a href=\"../../glossary.html#pre-pruning\" style=\"color:inherit; text-decoration:underline;\">Pre-pruning</a></li><li><a href=\"../../glossary.html#pure-leaf\" style=\"color:inherit; text-decoration:underline;\">Pure Leaf</a></li><li><a href=\"../../glossary.html#random-forest\" style=\"color:inherit; text-decoration:underline;\">Random Forest</a></li><li><a href=\"../../glossary.html#recursive-binary-splitting\" style=\"color:inherit; text-decoration:underline;\">Recursive Binary Splitting</a></li><li><a href=\"../../glossary.html#residual-sum-of-squares\" style=\"color:inherit; text-decoration:underline;\">Residual Sum of Squares</a></li><li><a href=\"../../glossary.html#root-node\" style=\"color:inherit; text-decoration:underline;\">Root Node</a></li><li><a href=\"../../glossary.html#stump\" style=\"color:inherit; text-decoration:underline;\">Stump</a></li><li><a href=\"../../glossary.html#tree-pruning\" style=\"color:inherit; text-decoration:underline;\">Tree Pruning</a></li><li><a href=\"../../glossary.html#tree-based-methods\" style=\"color:inherit; text-decoration:underline;\">Tree-based Methods</a></li><li><a href=\"../../glossary.html#weak-learners\" style=\"color:inherit; text-decoration:underline;\">Weak Learners</a></li>\n",
    "          </ul>\n",
    "        </div>\n",
    "        \n",
    "        <div>\n",
    "          <h4 style=\"margin:.25rem 0 .4rem; font-size:1rem; font-weight:700;\">\n",
    "            <span style=\"border-bottom:2px solid rgba(128,0,0,.55); padding-bottom:2px;\">Code</span>\n",
    "          </h4>\n",
    "          <ul style=\"margin:0; padding-left:1.2rem;\">\n",
    "            <li><a href=\"../../code-glossary.html#adaboostclassifier\" style=\"color:inherit; text-decoration:underline;\">AdaBoostClassifier</a></li><li><a href=\"../../code-glossary.html#baggingclassifier\" style=\"color:inherit; text-decoration:underline;\">BaggingClassifier</a></li><li><a href=\"../../code-glossary.html#decisiontreeclassifier\" style=\"color:inherit; text-decoration:underline;\">DecisionTreeClassifier</a></li><li><a href=\"../../code-glossary.html#gridsearchcv\" style=\"color:inherit; text-decoration:underline;\">GridSearchCV</a></li><li><a href=\"../../code-glossary.html#modelestimators\" style=\"color:inherit; text-decoration:underline;\">model.estimators_</a></li><li><a href=\"../../code-glossary.html#modelfeature-importances\" style=\"color:inherit; text-decoration:underline;\">model.feature_importances_</a></li><li><a href=\"../../code-glossary.html#pdcategoricalvaluescodes\" style=\"color:inherit; text-decoration:underline;\">pd.Categorical(values).codes</a></li><li><a href=\"../../code-glossary.html#plot-tree\" style=\"color:inherit; text-decoration:underline;\">plot_tree</a></li><li><a href=\"../../code-glossary.html#randomforestclassifier\" style=\"color:inherit; text-decoration:underline;\">RandomForestClassifier</a></li>\n",
    "          </ul>\n",
    "        </div>\n",
    "        \n",
    "  </div>\n",
    "</div>\n",
    "<!-- NEW_TERMS_END -->\n",
    "```\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}