536 lines
13 KiB
Plaintext
536 lines
13 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "bcf79585",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Exercice 2 - System evaluation"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "f642cedb",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Imports"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "9421a4e1",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import numpy as np"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "a0d67fa6",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Load data"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "5fe90672",
|
||
"metadata": {},
|
||
"source": [
|
||
"Define the path of the data file"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "ecd4a4cf",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"path = \"ex2-system-a.csv\""
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "246e7392",
|
||
"metadata": {},
|
||
"source": [
|
||
"Read the CSV file using `read_csv`"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "623096a5",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"dataset_a = pd.read_csv(path, sep=\";\", index_col=False, names=[\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"y_true\"])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "6f764c56",
|
||
"metadata": {},
|
||
"source": [
|
||
"Display first rows"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "c59a1651",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>0</th>\n",
|
||
" <th>1</th>\n",
|
||
" <th>2</th>\n",
|
||
" <th>3</th>\n",
|
||
" <th>4</th>\n",
|
||
" <th>5</th>\n",
|
||
" <th>6</th>\n",
|
||
" <th>7</th>\n",
|
||
" <th>8</th>\n",
|
||
" <th>9</th>\n",
|
||
" <th>y_true</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>5.348450e-08</td>\n",
|
||
" <td>7.493480e-10</td>\n",
|
||
" <td>8.083470e-07</td>\n",
|
||
" <td>2.082290e-05</td>\n",
|
||
" <td>5.222360e-10</td>\n",
|
||
" <td>2.330260e-08</td>\n",
|
||
" <td>5.241270e-12</td>\n",
|
||
" <td>9.999650e-01</td>\n",
|
||
" <td>4.808590e-07</td>\n",
|
||
" <td>0.000013</td>\n",
|
||
" <td>7</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>1.334270e-03</td>\n",
|
||
" <td>3.202960e-05</td>\n",
|
||
" <td>8.504280e-01</td>\n",
|
||
" <td>1.669090e-03</td>\n",
|
||
" <td>1.546460e-07</td>\n",
|
||
" <td>2.412940e-04</td>\n",
|
||
" <td>1.448280e-01</td>\n",
|
||
" <td>1.122810e-11</td>\n",
|
||
" <td>1.456330e-03</td>\n",
|
||
" <td>0.000011</td>\n",
|
||
" <td>2</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>3.643050e-06</td>\n",
|
||
" <td>9.962760e-01</td>\n",
|
||
" <td>2.045910e-03</td>\n",
|
||
" <td>4.210530e-04</td>\n",
|
||
" <td>2.194020e-05</td>\n",
|
||
" <td>1.644130e-05</td>\n",
|
||
" <td>2.838160e-04</td>\n",
|
||
" <td>3.722960e-04</td>\n",
|
||
" <td>5.150120e-04</td>\n",
|
||
" <td>0.000044</td>\n",
|
||
" <td>1</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>9.998200e-01</td>\n",
|
||
" <td>2.550390e-10</td>\n",
|
||
" <td>1.112010e-05</td>\n",
|
||
" <td>1.653200e-05</td>\n",
|
||
" <td>5.375730e-10</td>\n",
|
||
" <td>8.999750e-05</td>\n",
|
||
" <td>9.380920e-06</td>\n",
|
||
" <td>4.464470e-05</td>\n",
|
||
" <td>2.418440e-06</td>\n",
|
||
" <td>0.000006</td>\n",
|
||
" <td>0</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>2.092460e-08</td>\n",
|
||
" <td>7.464220e-08</td>\n",
|
||
" <td>3.560820e-05</td>\n",
|
||
" <td>5.496200e-07</td>\n",
|
||
" <td>9.988960e-01</td>\n",
|
||
" <td>3.070920e-08</td>\n",
|
||
" <td>2.346150e-04</td>\n",
|
||
" <td>9.748010e-07</td>\n",
|
||
" <td>1.071610e-06</td>\n",
|
||
" <td>0.000831</td>\n",
|
||
" <td>4</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" 0 1 2 3 4 \\\n",
|
||
"0 5.348450e-08 7.493480e-10 8.083470e-07 2.082290e-05 5.222360e-10 \n",
|
||
"1 1.334270e-03 3.202960e-05 8.504280e-01 1.669090e-03 1.546460e-07 \n",
|
||
"2 3.643050e-06 9.962760e-01 2.045910e-03 4.210530e-04 2.194020e-05 \n",
|
||
"3 9.998200e-01 2.550390e-10 1.112010e-05 1.653200e-05 5.375730e-10 \n",
|
||
"4 2.092460e-08 7.464220e-08 3.560820e-05 5.496200e-07 9.988960e-01 \n",
|
||
"\n",
|
||
" 5 6 7 8 9 y_true \n",
|
||
"0 2.330260e-08 5.241270e-12 9.999650e-01 4.808590e-07 0.000013 7 \n",
|
||
"1 2.412940e-04 1.448280e-01 1.122810e-11 1.456330e-03 0.000011 2 \n",
|
||
"2 1.644130e-05 2.838160e-04 3.722960e-04 5.150120e-04 0.000044 1 \n",
|
||
"3 8.999750e-05 9.380920e-06 4.464470e-05 2.418440e-06 0.000006 0 \n",
|
||
"4 3.070920e-08 2.346150e-04 9.748010e-07 1.071610e-06 0.000831 4 "
|
||
]
|
||
},
|
||
"execution_count": 4,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"dataset_a.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "41f040b0",
|
||
"metadata": {},
|
||
"source": [
|
||
"Store some useful statistics (class names + number of classes)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "fd0adce4",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"class_names = [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\"]\n",
|
||
"nb_classes = len(class_names)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "5a0ab85a",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Exercise's steps"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "66ae582e",
|
||
"metadata": {},
|
||
"source": [
|
||
"a) Write a function to take classification decisions on such outputs according to Bayes’rule."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "3c36b377",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def bayes_classification(df):\n",
|
||
" \"\"\"\n",
|
||
" Take classification decisions according to Bayes rule.\n",
|
||
" \n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" df : Pandas DataFrame of shape (n_samples, n_features + ground truth)\n",
|
||
" Dataset.\n",
|
||
" \n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" preds : Numpy array of shape (n_samples,)\n",
|
||
" Class labels for each data sample.\n",
|
||
" \"\"\"\n",
|
||
" # Your code here\n",
|
||
" pass"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "b5e8140b",
|
||
"metadata": {},
|
||
"source": [
|
||
"b) What is the overall error rate of the system ?"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "f3b21bfb",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Your code here: compute and print the error rate of the system"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "a4f0fa5f",
|
||
"metadata": {},
|
||
"source": [
|
||
"c) Compute and report the confusion matrix of the system."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"id": "bb106415",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def confusion_matrix(y_true, y_pred, n_classes):\n",
|
||
" \"\"\"\n",
|
||
" Compute the confusion matrix.\n",
|
||
" \n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" y_true : Numpy array of shape (n_samples,)\n",
|
||
" Ground truth.\n",
|
||
" y_pred : Numpy array of shape (n_samples,)\n",
|
||
" Predictions.\n",
|
||
" n_classes : Integer\n",
|
||
" Number of classes.\n",
|
||
" \n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" cm : Numpy array of shape (n_classes, n_classes)\n",
|
||
" Confusion matrix.\n",
|
||
" \"\"\"\n",
|
||
" # Your code here\n",
|
||
" pass"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "1b38e3a8",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Your code here: compute and print the confusion matrix"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "ed8db908",
|
||
"metadata": {},
|
||
"source": [
|
||
"d) What are the worst and best classes in terms of precision and recall ?"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "0e229ce0",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def precision_per_class(cm):\n",
|
||
" \"\"\"\n",
|
||
" Compute the precision per class.\n",
|
||
" \n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" cm : Numpy array of shape (n_classes, n_classes)\n",
|
||
" Confusion matrix.\n",
|
||
" \n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" precisions : Numpy array of shape (n_classes,)\n",
|
||
" Precision per class.\n",
|
||
" \"\"\"\n",
|
||
" # Your code here\n",
|
||
" pass"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 11,
|
||
"id": "95325772",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def recall_per_class(cm):\n",
|
||
" \"\"\"\n",
|
||
" Compute the recall per class.\n",
|
||
" \n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" cm : Numpy array of shape (n_classes, n_classes)\n",
|
||
" Confusion matrix.\n",
|
||
" \n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" recalls : Numpy array of shape (n_classes,)\n",
|
||
" Recall per class.\n",
|
||
" \"\"\"\n",
|
||
" # Your code here\n",
|
||
" pass"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 12,
|
||
"id": "a0fb19e3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Your code here: find and print the worst and best classes in terms of precision"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"id": "42c3edd8",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Your code here: find and print the worst and best classes in terms of recall"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "7ac6fe5d",
|
||
"metadata": {},
|
||
"source": [
|
||
"e) In file `ex1-system-b.csv` you find the output of a second system B. What is the best system between (a) and (b) in terms of error rate and F1."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 14,
|
||
"id": "b98c2545",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Your code here: load the data of the system B"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 15,
|
||
"id": "050091b9",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def system_accuracy(cm):\n",
|
||
" \"\"\"\n",
|
||
" Compute the system accuracy.\n",
|
||
" \n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" cm : Numpy array of shape (n_classes, n_classes)\n",
|
||
" Confusion matrix.\n",
|
||
" \n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" accuracy : Float\n",
|
||
" Accuracy of the system.\n",
|
||
" \"\"\"\n",
|
||
" # Your code here\n",
|
||
" pass"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 16,
|
||
"id": "adc0f138",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def system_f1_score(cm):\n",
|
||
" \"\"\"\n",
|
||
" Compute the system F1 score.\n",
|
||
" \n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" cm : Numpy array of shape (n_classes, n_classes)\n",
|
||
" Confusion matrix.\n",
|
||
" \n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" f1_score : Float\n",
|
||
" F1 score of the system.\n",
|
||
" \"\"\"\n",
|
||
" # Your code here\n",
|
||
" pass"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 17,
|
||
"id": "f1385c87",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Your code here: compute and print the accuracy and the F1 score of the system A"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"id": "50c64d08",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Your code here: compute and print the accuracy and the F1 score of the system B"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.7.11"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|