{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "18bdba27",
   "metadata": {},
   "source": [
    "# PW4 - Linear Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3629bfdf",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "id": "d6eb2aab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy.linalg import inv\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e69e5d68",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba14bf9f",
   "metadata": {},
   "source": [
    "Define the path to the file containing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "id": "5f3f924f",
   "metadata": {},
   "outputs": [],
   "source": [
    "datafile = \"lausanne-appart.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adfab574",
   "metadata": {},
   "source": [
    "Read the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "id": "5108e239",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(datafile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a97d4cc",
   "metadata": {},
   "source": [
    "Display first rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "id": "34222aff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>living_area</th>\n",
       "      <th>nb_rooms</th>\n",
       "      <th>rent_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>69</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>95</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>21</td>\n",
       "      <td>1.5</td>\n",
       "      <td>685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33</td>\n",
       "      <td>1.5</td>\n",
       "      <td>830</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   living_area  nb_rooms  rent_price\n",
       "0           69       3.0        1810\n",
       "1           95       3.5        2945\n",
       "2           21       1.5         685\n",
       "3           20       1.0         720\n",
       "4           33       1.5         830"
      ]
     },
     "execution_count": 457,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ec4b8eb",
   "metadata": {},
   "source": [
    "## Exercise 1 - Visualize the data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbf6b9cf",
   "metadata": {},
   "source": [
    "a) Plot a histogram to visualize the distribution of the renting price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "id": "ad07d4cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 14004 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "\n",
    "plt.hist(dataset[\"rent_price\"])\n",
    "plt.xlabel(\"Rent price\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "602770ae",
   "metadata": {},
   "source": [
    "b) Plot a histogram to visualize the distribution of the living area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 459,
   "id": "ac0c9f07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12584 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "\n",
    "plt.hist(dataset[\"living_area\"])\n",
    "plt.xlabel(\"Living Area\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9a954dd",
   "metadata": {},
   "source": [
    "c) Plot a scatter plot of renting price as a function of living area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "id": "6d015af9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31436 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "\n",
    "plt.scatter(dataset[\"living_area\"], dataset[\"rent_price\"])\n",
    "plt.xlabel(\"Living Area\")\n",
    "plt.ylabel(\"Rent price\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c787c4b",
   "metadata": {},
   "source": [
    "## Exercise 2 - Normal equations for linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "859bbd89",
   "metadata": {},
   "source": [
    "a) Implement the closed form solution to this problem using the following normal equation:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf1d5eda",
   "metadata": {},
   "source": [
    "<div><div style=\"display: table-cell; width: 100%;\"><center>$\\theta = (X^{T}X)^{-1}X^{T}\\vec{y}$</center></div><div style=\"display: table-cell; width: 100%;\">$(3)$</div></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3c8e4c1",
   "metadata": {},
   "source": [
    "Assuming $x$ is the living area and $y$ is the renting price. Use `numpy` for the vector operations. Plot the computed line on top of the scatter plot of exercise 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "id": "19ffbbd9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theta : [240.07710727  26.33242457]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 37760 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "X = dataset[\"living_area\"].to_frame()\n",
    "X.insert(0, \"x0\", 1) # add the column full of ones at the start, this represents x0\n",
    "X = np.array(X)\n",
    "\n",
    "y = np.array(dataset[\"rent_price\"])\n",
    "\n",
    "theta = np.linalg.inv(X.T @ X) @ X.T @ y \n",
    "\n",
    "print(f\"Theta : {theta}\")\n",
    "\n",
    "\n",
    "plt.scatter(dataset[\"living_area\"], dataset[\"rent_price\"])\n",
    "plt.xlabel(\"Living Area\")\n",
    "plt.ylabel(\"Rent price\")\n",
    "\n",
    "y_regr = theta[0] + theta[1] * X[:,1]\n",
    "plt.plot(X[:,1], y_regr, color='red', label='Regression line')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccc5d7b9",
   "metadata": {},
   "source": [
    "b) Compute the overall cost value according to the following equation:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31a38211",
   "metadata": {},
   "source": [
    "<div><div style=\"display: table-cell; width: 100%;\"><center>$J(\\theta) = \\frac{1}{2N} \\sum_{n=1}^{N} (h_{\\theta}(\\mathbf{x}_{n}) - y_{n})^{2}$</center></div><div style=\"display: table-cell; width: 100%;\">$(2)$</div></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "id": "db65a2fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Overall cost : 138034.95779787414\n"
     ]
    }
   ],
   "source": [
    "# Your code here\n",
    "def cost_J(y_pred, y):\n",
    "    return (1/(2*len(y))) * np.sum((y_pred - y)**2)\n",
    "\n",
    "y_pred = X @ theta\n",
    "# print(y_pred)\n",
    "# print(X)\n",
    "closed_form_cost = cost_J(y_pred, y)\n",
    "print(f\" Overall cost : {closed_form_cost}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38fed9aa",
   "metadata": {},
   "source": [
    "## Exercise 3 - Gradient descent for linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d946ac48",
   "metadata": {},
   "source": [
    "Implement the _full batch_ gradient descent algorithm for the previous problem. As seen in the\n",
    "theory, the update rules are:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d1618d6",
   "metadata": {},
   "source": [
    "<div><div style=\"display: table-cell; width: 100%;\"><center>$\\theta_{0} \\leftarrow \\theta_{0} - \\alpha \\frac{1}{N} \\sum_{n=1}^{N} (h_{\\theta}(\\mathbf{x}_{n}) - y_{n})$</center></div><div style=\"display: table-cell; width: 100%;\">$(6)$</div></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c0049d3",
   "metadata": {},
   "source": [
    "<div><div style=\"display: table-cell; width: 100%;\"><center>$\\theta_{1} \\leftarrow \\theta_{1} - \\alpha \\frac{1}{N} \\sum_{n=1}^{N} (h_{\\theta}(\\mathbf{x}_{n}) - y_{n}) x_{n,1}$</center></div><div style=\"display: table-cell; width: 100%;\">$(7)$</div></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e0f8c42",
   "metadata": {},
   "source": [
    "**Remark** You need to iterate several times over the training set. If you have problems of convergence, you need to use a smaller value of $\\alpha$. Values such as $0.000001$ are common."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "id": "9ca79a3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theta = [240.07710713  26.33242457]\n",
      "Cost = 138034.95779787414\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 18060 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "\n",
    "def gradient_descent(X, y, max_iter, alpha):\n",
    "    theta = np.zeros((2))\n",
    "    costs = []\n",
    "\n",
    "    for i in range(0, max_iter): # we could also use a delta on the cost function with a while to stop when the cost is not improving but we wanted to see the curve after x iterations\n",
    "        y_pred = X @ theta\n",
    "        theta[0] = theta[0] - alpha * (1/len(y)) * np.sum(y_pred - y)\n",
    "        theta[1] = theta[1] - alpha * (1/len(y)) * np.sum((y_pred - y) * X[:,1])  \n",
    "\n",
    "        c = cost_J(y_pred, y) # to plot the costs over iters\n",
    "        costs.append(c)\n",
    "\n",
    "    y_pred = X @ theta\n",
    "    print(f\"Theta = {theta}\")\n",
    "    print(f\"Cost = {cost_J(y_pred, y)}\")\n",
    "\n",
    "    plt.plot(range(0, len(costs)), costs)\n",
    "    plt.title(\"Cost over iterations\")\n",
    "    plt.show()\n",
    "\n",
    "gradient_descent(X, y, max_iter = 1000000, alpha = 0.0001) # i got the best results with those parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da5121b6",
   "metadata": {},
   "source": [
    "a) Plot the cost value (Equation 2) as a function of the iterations. What do you observe?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "id": "5a34ad43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 18060 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "plt.plot(range(0, len(costs)), costs)\n",
    "plt.title(\"Cost over iterations\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63917be0",
   "metadata": {},
   "source": [
    "In this scenario, we can clearly observe a plateau of the cost function. This means that we have reached a local minimum while trying to minimise the cost."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5903f37",
   "metadata": {},
   "source": [
    "b) Imagine a stopping criterion, i.e. when do we stop iterating on the training set?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93905b85",
   "metadata": {},
   "source": [
    "The stopping criterion can be a small delta between each calculations of the cost. Like if the cost did not change of at least delta, this means we are in a plateau and should stop."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7667bec1",
   "metadata": {},
   "source": [
    "c) Plot the computed line $h_{\\theta}(\\mathbf{x})$ on top of the scatter plot of exercise 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "id": "e80c04b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37760 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(dataset[\"living_area\"], dataset[\"rent_price\"])\n",
    "plt.xlabel(\"Living Area\")\n",
    "plt.ylabel(\"Rent price\")\n",
    "\n",
    "y_regr = theta[0] + theta[1] * X[:,1]\n",
    "plt.plot(X[:,1], y_regr, color='red', label='Regression line')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1fdf27b",
   "metadata": {},
   "source": [
    "d) Compute the final cost value according to Equation 2 and compare it to the one of exercise 2. What can you conclude?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "id": "5e594f32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best cost value is inf, the exercice 2 cost value is 138034.95779787414\n"
     ]
    }
   ],
   "source": [
    "# Your code here\n",
    "\n",
    "print(f\"The best cost value is {best_cost}, the exercice 2 cost value is {closed_form_cost}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd10b91b",
   "metadata": {},
   "source": [
    "We took some time to experiment with the alpha and max_iter values to try to get the best possible result. It seems that the closed form formula is the best one but we can get really close by tweaking those two values. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54727ff1",
   "metadata": {},
   "source": [
    "## Exercise 4 - Stochastic gradient descent for linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32a00957",
   "metadata": {},
   "source": [
    "Implement the stochastic gradient descent algorithm for the previous problem. As seen in the theory, the update rules are:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "636920d2",
   "metadata": {},
   "source": [
    "<div><div style=\"display: table-cell; width: 100%;\"><center>$\\theta_{i} \\leftarrow \\theta_{i} - \\alpha (h_{\\theta}(\\mathbf{x}_{n}) - y_{n}) x_{n,i}$</center></div><div style=\"display: table-cell; width: 100%;\">$(8)$</div></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 467,
   "id": "3f693388",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theta = [ 0.33101883 28.20026525]\n",
      "Cost = 145028.5077448914\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 24808 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "import random\n",
    "def stochastic_gradient_descent(X, y, max_iter, alpha):\n",
    "    \n",
    "    theta = np.zeros((2))\n",
    "\n",
    "    costs = []\n",
    "\n",
    "\n",
    "    for i in range(0, max_iter): # we could also use a delta on the cost function with a while to stop when the cost is not improving but we wanted to see the curve after x iterations\n",
    "        rand = int(random.random() * X.shape[0])\n",
    "        y_pred_stoch = X[rand] @ theta\n",
    "        theta[0] = theta[0] - alpha * (y_pred_stoch - y[rand])\n",
    "        theta[1] = theta[1] - alpha * (y_pred_stoch - y[rand]) * X[rand,1]\n",
    "\n",
    "        y_pred = X @ theta\n",
    "        costs.append(cost_J(y_pred, y))\n",
    "\n",
    "\n",
    "    y_pred = X @ theta\n",
    "    print(f\"Theta = {theta}\")\n",
    "    print(f\"Cost = {cost_J(y_pred, y)}\")\n",
    "\n",
    "    plt.plot(range(0, len(costs)), costs)\n",
    "    plt.title(\"Cost over iterations\")\n",
    "    plt.show()\n",
    "    return theta\n",
    "\n",
    "theta = stochastic_gradient_descent(X, y, max_iter = 1000, alpha = 0.000001)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb73d7c1",
   "metadata": {},
   "source": [
    "a) Plot the computed line $h_{\\theta}(\\mathbf{x})$ on top of the scatter plot of exercise 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "id": "f73c6af3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37188 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Your code here\n",
    "plt.scatter(dataset[\"living_area\"], dataset[\"rent_price\"])\n",
    "plt.xlabel(\"Living Area\")\n",
    "plt.ylabel(\"Rent price\")\n",
    "\n",
    "y_regr = theta[0] + theta[1] * X[:,1]\n",
    "plt.plot(X[:,1], y_regr, color='red', label='Regression line')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16e1eec7",
   "metadata": {},
   "source": [
    "b) How many samples do you need to visit for reaching the convergence?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd7870eb",
   "metadata": {},
   "source": [
    "The biggest improvements are made before the first 300ish iterations but smaller improvements still occur after that. If we were very limited in cpu time, we would tell that the convergence vas reached around 300 samples"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c185d7bc",
   "metadata": {},
   "source": [
    "c) What kind of stopping criterion could we use here?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "235f3ac0",
   "metadata": {},
   "source": [
    "We would use the delta of the global cost function. When we compute the cost function, we can stop the iterations when the result is not improving more than a set delta."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e7ce003",
   "metadata": {},
   "source": [
    "d) Compute the final cost value according to Equation 2 and compare it to the one of exercise 2 and 3. What can you conclude?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 469,
   "id": "dbcb6f22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theta = [ 0.33101883 28.20026525]\n",
      "Cost = 145028.5077448914\n"
     ]
    }
   ],
   "source": [
    "# Your code here\n",
    "y_pred = X @ theta\n",
    "print(f\"Theta = {theta}\")\n",
    "print(f\"Cost = {cost_J(y_pred, y)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27a8e7fd",
   "metadata": {},
   "source": [
    "The result is really close to both of the other methods but the stochastic is way faster because we dont loop trough the whole dataset for each update. The stochastic might have a slightly worse final result because it is harder to find the exact perfect values but its speed and efficency compensates this."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8a85676",
   "metadata": {},
   "source": [
    "## Exercise 5 - Review questions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84f82d3d",
   "metadata": {},
   "source": [
    "a) The linear regression has a mathematical _closed form_ solution. Then, in which conditions would we prefer a gradient descent algorithm to compute the regression model?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eecbe019",
   "metadata": {},
   "source": [
    "The closed form solution is great to have to best value possible, but it is very compute hungry. The inverse of a really big matrix is not something simple to do. So, the gradient descent is better when compute time is limited."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3696b258",
   "metadata": {},
   "source": [
    "b) Outliers in a data set can be defined as values that are out of the \"usual\" range in comparison with other values. They typically come from noise or anomalies in the data capturing process. What is the impact of an outlier in the stochastic gradient descent process? What if we have many outliers? _Hint_: look at the equation of the MSE, and to the equation of the update rule."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4873219d",
   "metadata": {},
   "source": [
    "Since for each iterations we update theta based on a random point, the outliers could significantly slow the congergence. If there is a lot of them, each update of the thetha values will not decrease the total cost value. This means that, if we are in a while loop searching to improve the cost value, it might either cause more loops to achieve the desired goal or cause the while loop to exit because the cost value is not getting smaller, but bigger."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22dce221",
   "metadata": {},
   "source": [
    "c) In the case of stochastic gradient descent, what is the danger of having a too large or too small $\\alpha$ value? Could you think of a better (more advanced) strategy as the one stated in Slide 36?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1536554",
   "metadata": {},
   "source": [
    "A too small alpha value would make the descent too slow and remove the speed advantage of this method. An alpha too big would cause the theta values to jump from an extreme to another. This would be highly susceptible to outliers. To mitigate this, we could apply the same logic, but pick a small number of samples (mini-batch) rather than a single one for each operations. This would reduce the impact of outliers while keeping the algorithm faster than the full one."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07bdad8b",
   "metadata": {},
   "source": [
    "d) Let's assume we expect that the target variable $y$ has a dependency to the square and to the cube of one of the feature $x_d$ in our multi-variable training set $(x_1, \\ldots, x_d, \\ldots, x_D)$. How would you proceed? Do we need to take precautions in terms of numerical stability?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4f242c5",
   "metadata": {},
   "source": [
    "If we have a dependency to the square and to the cube of the feature f1, we would augment the matrix to reflect this :  h(f) = theta0 + theta1xf1 + theta2xf1² + theta3xf1³. Yes, we need to take precautions because f1³ could mean huge numbers and cause overflows. The solutions would be to scale down each value to keep the relations between them but reduce their overall values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7cf619b",
   "metadata": {},
   "source": [
    "e) _Advanced_. Could we use a descent algorithm without computing the gradient? If yes, give a pseudo code of the algorithm to find $(\\theta_{0}, \\theta_{1})$ using a linear regression $h_{\\theta}(x) = \\theta_0 + \\theta_1 x$."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89317af1",
   "metadata": {},
   "source": [
    "_Your answer here_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b0997fa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8ce59c65",
   "metadata": {},
   "source": [
    "# Exercice 6 Optional – Mini-batch gradient descent for linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "id": "8604ed56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stochastic : \n",
      "Theta = [ 0.31909445 28.77358351]\n",
      "Cost = 144217.6608921882\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 25076 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch of 1 : \n",
      "Theta = [ 0.30493246 27.72933782]\n",
      "Cost = 147758.42400290794\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 24460 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch of 10 : \n",
      "Theta = [ 0.32782064 28.84157514]\n",
      "Cost = 144303.73303813871\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 23960 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([ 0.32782064, 28.84157514])"
      ]
     },
     "execution_count": 470,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def batch_gradient_descent(X, y, batch_size, max_iter, alpha):\n",
    "    theta = np.zeros((2))\n",
    "\n",
    "    costs = []\n",
    "\n",
    "    for i in range(0, max_iter): # we could also use a delta on the cost function with a while to stop when the cost is not improving but we wanted to see the curve after x iterations\n",
    "        batch_idx = np.random.choice(y.shape[0], batch_size)\n",
    "        y_stoch = y[batch_idx]\n",
    "        y_pred_stoch = X[batch_idx, :] @ theta\n",
    "\n",
    "        theta[0] = theta[0] - alpha * (1/len(y_stoch)) * np.sum(y_pred_stoch - y_stoch)\n",
    "        theta[1] = theta[1] - alpha * (1/len(y_stoch)) * np.sum((y_pred_stoch - y_stoch) * X[batch_idx, 1])\n",
    "\n",
    "        y_pred = X @ theta\n",
    "        costs.append(cost_J(y_pred, y))\n",
    "\n",
    "\n",
    "    y_pred = X @ theta\n",
    "    print(f\"Theta = {theta}\")\n",
    "    print(f\"Cost = {cost_J(y_pred, y)}\")\n",
    "\n",
    "    plt.plot(range(0, len(costs)), costs)\n",
    "    plt.title(\"Cost over iterations\")\n",
    "    plt.show()\n",
    "    return theta\n",
    "\n",
    "\n",
    "print(\"Stochastic : \")\n",
    "stochastic_gradient_descent(X, y, max_iter = 1000, alpha=0.000001)\n",
    "print(\"batch of 1 : \")\n",
    "batch_gradient_descent(X, y, batch_size = 1, max_iter = 1000, alpha = 0.000001)\n",
    "print(\"batch of 10 : \")\n",
    "batch_gradient_descent(X, y, batch_size = 10, max_iter = 1000, alpha = 0.000001)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35e0d314",
   "metadata": {},
   "source": [
    "# Exercice 7 Optional – multi-variable linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de60cd7e",
   "metadata": {},
   "source": [
    "## A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 471,
   "id": "928deeaa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theta = [2301.42126052 1049.63985381   62.36135978]\n",
      "Cost = 134038.2148043482\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 24416 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 38720 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def multivariate_batch_gradient_descent(X, y, batch_size, max_iter, alpha):\n",
    "    theta = np.zeros((X.shape[1]))\n",
    "\n",
    "    costs = []\n",
    "\n",
    "    for i in range(0, max_iter): # we could also use a delta on the cost function with a while to stop when the cost is not improving but we wanted to see the curve after x iterations\n",
    "        batch_idx = np.random.choice(y.shape[0], batch_size)\n",
    "        y_stoch = y[batch_idx]\n",
    "        y_pred_stoch = X[batch_idx, :] @ theta\n",
    "\n",
    "        for j in range(0, X.shape[1]):\n",
    "            theta[j] = theta[j] - alpha * (1/len(y_stoch)) * np.sum((y_pred_stoch - y_stoch) * X[batch_idx, j])\n",
    "\n",
    "        y_pred = X @ theta\n",
    "        costs.append(cost_J(y_pred, y))\n",
    "\n",
    "\n",
    "    y_pred = X @ theta\n",
    "    print(f\"Theta = {theta}\")\n",
    "    print(f\"Cost = {cost_J(y_pred, y)}\")\n",
    "\n",
    "    plt.plot(range(0, len(costs)), costs)\n",
    "    plt.title(\"Cost over iterations\")\n",
    "    plt.show()\n",
    "\n",
    "    return theta\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# need to normalize the data or else overflows will occur\n",
    "living = dataset[\"living_area\"].to_frame()\n",
    "mean_living = living.mean()\n",
    "std_living = living.std()\n",
    "living_normalized = (living - mean_living) / std_living\n",
    "\n",
    "X_norm = living_normalized\n",
    "X_norm.insert(0, \"x0\", 1) # add the column full of ones at the start, this represents x0\n",
    "X_norm.insert(2, \"x2\", X_norm.iloc[:,1]**2) # add the squared value of column 1 to column 2\n",
    "X_norm = np.array(X_norm)\n",
    "\n",
    "y = np.array(dataset[\"rent_price\"])\n",
    "\n",
    "theta = multivariate_batch_gradient_descent(X_norm, y, batch_size=10, max_iter = 10000, alpha=0.001)\n",
    "\n",
    "plt.scatter(dataset[\"living_area\"], dataset[\"rent_price\"])\n",
    "plt.xlabel(\"Living Area\")\n",
    "plt.ylabel(\"Rent price\")\n",
    "\n",
    "y_regr = theta[0] + theta[1] * X_norm[:,1] + theta[2] * X_norm[:,2]\n",
    "plt.plot(dataset[\"living_area\"], y_regr, color='red', label='Regression line')\n",
    "\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64f2201b",
   "metadata": {},
   "source": [
    "The convergence is still happening. We observed that we could use a way bigger alpha to achieve the same results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a48af614",
   "metadata": {},
   "source": [
    "## B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "id": "dcfa5986",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theta = [ 1.06588497  0.32970238 28.86894189]\n",
      "Cost = 144422.06094103807\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 24024 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 37380 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = dataset[\"living_area\"].to_frame()\n",
    "X.insert(0, \"x0\", 1) # add the column full of ones at the start, this represents x0\n",
    "X.insert(0, \"x2\", dataset[\"nb_rooms\"].to_frame()) # add the column full of ones at the start, this represents x0\n",
    "X = np.array(X)\n",
    "\n",
    "y = np.array(dataset[\"rent_price\"])\n",
    "\n",
    "theta = multivariate_batch_gradient_descent(X, y, batch_size=10, max_iter = 1000, alpha=0.000001)\n",
    "\n",
    "plt.scatter(dataset[\"living_area\"], dataset[\"rent_price\"])\n",
    "plt.xlabel(\"Living Area\")\n",
    "plt.ylabel(\"Rent price\")\n",
    "\n",
    "y_regr = theta[0] + theta[1] * X[:,1] + theta[2] * X[:,2]\n",
    "plt.plot(dataset[\"living_area\"], y_regr, color='red', label='Regression line')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed1e2d4f",
   "metadata": {},
   "source": [
    "The algorithm still manages to converge, but no clear improvements is seen here"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
