{ "cells": [ { "cell_type": "markdown", "id": "f987082c", "metadata": {}, "source": [ "# Perceptron\n", "\n", "Inverting the image takes the accuracy insanely high. like went \n", "\n", "from (25, 14) to (58, 19) in PLA\n", "\n", "from (3, 3) to (94, 46) in MLP" ] }, { "cell_type": "markdown", "id": "5d3a7749", "metadata": {}, "source": [ "## 1. Preprocessing" ] }, { "cell_type": "code", "execution_count": 1, "id": "02e0b100", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
imagelabel
222Img/img005-003.png4
1635Img/img030-041.pngT
355Img/img007-026.png6
2729Img/img050-035.pngn
1814Img/img033-055.pngW
\n", "
" ], "text/plain": [ " image label\n", "222 Img/img005-003.png 4\n", "1635 Img/img030-041.png T\n", "355 Img/img007-026.png 6\n", "2729 Img/img050-035.png n\n", "1814 Img/img033-055.png W" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "from PIL import Image\n", "from tqdm import tqdm\n", "\n", "df = pd.read_csv('english.csv')\n", "df.sample(5)" ] }, { "cell_type": "code", "execution_count": 2, "id": "7af60014", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 3410/3410 [00:17<00:00, 192.02it/s]\n" ] } ], "source": [ "images = []\n", "\n", "for path in tqdm(df[\"image\"]):\n", " with Image.open(path) as img:\n", " # Resize\n", " img_grey = img.convert('L').resize((28,28))\n", " # flatten\n", " # normalised = np.array(img_grey, dtype=np.float32).flatten() / 255.0\n", " # inverted\n", " normalised = 1.0 - (np.array(img_grey, dtype=np.float32).flatten() / 255.0)\n", " images.append(normalised)" ] }, { "cell_type": "code", "execution_count": 3, "id": "294e3b0f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Index(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D',\n", " 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',\n", " 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',\n", " 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',\n", " 'u', 'v', 'w', 'x', 'y', 'z'],\n", " dtype='str')\n", "[ 0 0 0 ... 61 61 61]\n" ] } ], "source": [ "X = np.array(images)\n", "X = np.c_[np.ones(X.shape[0]), X]\n", "\n", "# label to numbers\n", "y, label = pd.factorize(df[\"label\"])\n", "print(label)\n", "print(y)" ] }, { "cell_type": "code", "execution_count": 4, "id": "48dae170", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "X_train: (2728, 785), y_train: (2728,)\n", "X_test: (682, 785), y_test: (682,)\n", "Classes: 62\n" ] } ], "source": [ "# train test split\n", "\n", "train_split = 0.8\n", "\n", "indices = np.random.permutation(len(X))\n", "split_idx = int(train_split * len(X))\n", "\n", "train_idx, test_idx = indices[:split_idx], indices[split_idx:]\n", "X_train, X_test = X[train_idx], X[test_idx]\n", "y_train, y_test = y[train_idx], y[test_idx]\n", "\n", "print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n", "print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")\n", "print(f\"Classes: {len(label)}\")" ] }, { "cell_type": "markdown", "id": "9031b9a8", "metadata": {}, "source": [ "# 2. PLA" ] }, { "cell_type": "code", "execution_count": 5, "id": "608b0436", "metadata": {}, "outputs": [], "source": [ "def step_activation(z):\n", " return np.where(z >= 0, 1, -1)" ] }, { "cell_type": "code", "execution_count": 6, "id": "05710aed", "metadata": {}, "outputs": [], "source": [ "# Single training\n", "def train_binary_pla(X, y_for_one, lr=0.1, max_epochs=50):\n", " n_samples, n_features = X.shape\n", " w = np.zeros(n_features)\n", "\n", " for epoch in range(max_epochs):\n", " errors = 0\n", "\n", " for i in range(n_samples):\n", " z = np.dot(X[i], w)\n", " y_pred = step_activation(z)\n", "\n", " if y_pred != y_for_one[i]:\n", " w += lr * y_for_one[i] * X[i]\n", " errors += 1\n", "\n", " # Early stopping if data is linearly separable\n", " if errors == 0:\n", " break\n", " return w\n", "\n", "# multiclass\n", "def fit_ovr_pla(X, y, num_classes, lr=0.1, max_epochs=50):\n", " n_features = X.shape[1]\n", " W = np.zeros((num_classes, n_features))\n", "\n", " for clas in tqdm(range(num_classes)):\n", " y_binary = np.where(y == clas, 1, -1)\n", " W[clas] = train_binary_pla(X, y_binary, lr=lr, max_epochs=max_epochs)\n", "\n", " return W\n", "\n", "def predict(X, W):\n", " scores = np.dot(X, W.T)\n", " return np.argmax(scores, axis=1)\n", "\n", "\n", "def accuracy(y_true, y_predict):\n", " return np.mean(y_true == y_predict) * 100" ] }, { "cell_type": "code", "execution_count": 7, "id": "ef77f4fe", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 62/62 [00:14<00:00, 4.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Train Accuracy: 61.69%\n", "Test Accuracy: 19.65%\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# Train One-vs-Rest PLA\n", "num_classes = len(np.unique(y_train))\n", "W = fit_ovr_pla(X_train, y_train, num_classes=num_classes, lr=0.1, max_epochs=50)\n", "\n", "# Evaluate on Train and Test sets\n", "train_preds = predict(X_train, W)\n", "test_preds = predict(X_test, W)\n", "\n", "print(f\"Train Accuracy: {accuracy(y_train, train_preds):.2f}%\")\n", "print(f\"Test Accuracy: {accuracy(y_test, test_preds):.2f}%\")" ] }, { "cell_type": "markdown", "id": "8a3af236", "metadata": {}, "source": [ "# 3. MLP" ] }, { "cell_type": "code", "execution_count": 18, "id": "eae31124", "metadata": {}, "outputs": [], "source": [ "# removing bias\n", "\n", "X_train_mlp = X_train[:, 1:] if X_train.shape[1] == 785 else X_train\n", "X_test_mlp = X_test[:, 1:] if X_test.shape[1] == 785 else X_test" ] }, { "cell_type": "code", "execution_count": 19, "id": "c1916b9f", "metadata": {}, "outputs": [], "source": [ "# One-hot encode targets for cross-entropy training\n", "n_classes = len(np.unique(y_train))\n", "Y_train_one_hot = np.eye(num_classes)[y_train]" ] }, { "cell_type": "code", "execution_count": 20, "id": "fad84919", "metadata": {}, "outputs": [], "source": [ "def relu(Z):\n", " return np.maximum(0, Z)\n", "\n", "def relu_backward(dZ, Z):\n", " return dZ * (Z > 0)\n", "\n", "def softmax(Z):\n", " exp_Z = np.exp(Z - np.max(Z, axis=1, keepdims=True))\n", " return exp_Z / np.sum(exp_Z, axis=1, keepdims=True)\n", "\n", "def init_mlp_layers(layer_dims, activation=\"ReLU\"):\n", " np.random.seed(42)\n", " params = {}\n", " L = len(layer_dims) # Total layers including input & output\n", "\n", " for l in range(1, L):\n", " n_in, n_out = layer_dims[l - 1], layer_dims[l]\n", "\n", " # He init for ReLU, Xavier/Glorot for Tanh\n", " std = (\n", " np.sqrt(2.0 / n_in)\n", " if activation == \"ReLU\"\n", " else np.sqrt(1.0 / n_in)\n", " )\n", " params[f\"W{l}\"] = np.random.randn(n_in, n_out) * std\n", " params[f\"b{l}\"] = np.zeros((1, n_out))\n", "\n", " return params" ] }, { "cell_type": "code", "execution_count": 21, "id": "65c8bbe7", "metadata": {}, "outputs": [], "source": [ "def forward_pass_general(X, params, activation=\"ReLU\"):\n", " L = len(params) // 2\n", " cache = {\"A0\": X}\n", " act_func = relu if activation == \"ReLU\" else tanh\n", "\n", " for l in range(1, L):\n", " cache[f\"Z{l}\"] = (\n", " np.dot(cache[f\"A{l-1}\"], params[f\"W{l}\"]) + params[f\"b{l}\"]\n", " )\n", " cache[f\"A{l}\"] = act_func(cache[f\"Z{l}\"])\n", "\n", " # Output layer with Softmax\n", " cache[f\"Z{L}\"] = np.dot(cache[f\"A{L-1}\"], params[f\"W{L}\"]) + params[f\"b{L}\"]\n", " cache[f\"A{L}\"] = softmax(cache[f\"Z{L}\"])\n", "\n", " return cache[f\"A{L}\"], cache\n", "\n", "\n", "def backward_pass_general(X, Y, cache, params, activation=\"ReLU\"):\n", " L = len(params) // 2\n", " m = X.shape[0]\n", " grads = {}\n", " act_backward = relu_backward if activation == \"ReLU\" else tanh_backward\n", "\n", " # Output Layer Gradients (Cross-Entropy + Softmax)\n", " dZ = (cache[f\"A{L}\"] - Y) / m\n", " grads[f\"dW{L}\"] = np.dot(cache[f\"A{L-1}\"].T, dZ)\n", " grads[f\"db{L}\"] = np.sum(dZ, axis=0, keepdims=True)\n", "\n", " # Hidden Layers Gradients\n", " for l in range(L - 1, 0, -1):\n", " dA = np.dot(dZ, params[f\"W{l+1}\"].T)\n", " dZ = act_backward(dA, cache[f\"Z{l}\"])\n", " grads[f\"dW{l}\"] = np.dot(cache[f\"A{l-1}\"].T, dZ)\n", " grads[f\"db{l}\"] = np.sum(dZ, axis=0, keepdims=True)\n", "\n", " return grads" ] }, { "cell_type": "code", "execution_count": 22, "id": "740018b3", "metadata": {}, "outputs": [], "source": [ "def train_mlp(\n", " X,\n", " Y_one_hot,\n", " hidden_dims=[128],\n", " activation=\"ReLU\",\n", " optimizer=\"SGD\",\n", " lr=0.01,\n", " batch_size=32,\n", " epochs=50,\n", "):\n", " n_samples, n_features = X.shape\n", " n_classes = Y_one_hot.shape[1]\n", " layer_dims = [n_features] + hidden_dims + [n_classes]\n", " params = init_mlp_layers(layer_dims, activation=activation)\n", "\n", " # Adam state initialization\n", " L = len(layer_dims) - 1\n", " m_opt, v_opt = {}, {}\n", " beta1, beta2, eps = 0.9, 0.999, 1e-8\n", " t = 0\n", "\n", " if optimizer == \"Adam\":\n", " for l in range(1, L + 1):\n", " m_opt[f\"W{l}\"] = np.zeros_like(params[f\"W{l}\"])\n", " m_opt[f\"b{l}\"] = np.zeros_like(params[f\"b{l}\"])\n", " v_opt[f\"W{l}\"] = np.zeros_like(params[f\"W{l}\"])\n", " v_opt[f\"b{l}\"] = np.zeros_like(params[f\"b{l}\"])\n", "\n", " for epoch in tqdm(range(epochs)):\n", " perm = np.random.permutation(n_samples)\n", " X_shuff = X[perm]\n", " Y_shuff = Y_one_hot[perm]\n", "\n", " for i in range(0, n_samples, batch_size):\n", " X_b = X_shuff[i : i + batch_size]\n", " Y_b = Y_shuff[i : i + batch_size]\n", "\n", " _, cache = forward_pass_general(\n", " X_b, params, activation=activation\n", " )\n", " grads = backward_pass_general(\n", " X_b, Y_b, cache, params, activation=activation\n", " )\n", "\n", " # Optimization Step\n", " if optimizer == \"SGD\":\n", " for l in range(1, L + 1):\n", " params[f\"W{l}\"] -= lr * grads[f\"dW{l}\"]\n", " params[f\"b{l}\"] -= lr * grads[f\"db{l}\"]\n", "\n", " elif optimizer == \"Adam\":\n", " t += 1\n", " for l in range(1, L + 1):\n", " for p in [\"W\", \"b\"]:\n", " key = f\"{p}{l}\"\n", " # First and second moment estimates\n", " m_opt[key] = (\n", " beta1 * m_opt[key] + (1 - beta1) * grads[f\"d{key}\"]\n", " )\n", " v_opt[key] = beta2 * v_opt[key] + (1 - beta2) * (\n", " grads[f\"d{key}\"] ** 2\n", " )\n", "\n", " # Bias-corrected estimates\n", " m_hat = m_opt[key] / (1 - beta1**t)\n", " v_hat = v_opt[key] / (1 - beta2**t)\n", "\n", " params[key] -= lr * m_hat / (np.sqrt(v_hat) + eps)\n", "\n", " return params\n", "\n", "def predict_mlp(X, params, activation=\"ReLU\"):\n", " A_out, _ = forward_pass_general(X, params, activation=activation)\n", " return np.argmax(A_out, axis=1)" ] }, { "cell_type": "code", "execution_count": 24, "id": "ee690b5f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 60/60 [00:02<00:00, 23.04it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "MLP Train Accuracy: 46.30%\n", "MLP Test Accuracy: 35.04%\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "trained_params = train_mlp_general(\n", " X_train_mlp,\n", " Y_train_one_hot,\n", " hidden_dims=[128],\n", " activation=\"ReLU\",\n", " optimizer=\"SGD\",\n", " lr=0.01,\n", " batch_size=32,\n", " epochs=60\n", " )\n", "y_pred_train = predict_mlp(X_train_mlp, trained_params, activation=\"ReLU\")\n", "y_pred_test = predict_mlp(X_test_mlp, trained_params, activation=\"ReLU\")\n", "\n", "\n", "print(f\"MLP Train Accuracy: {accuracy(y_train, y_pred_train):.2f}%\")\n", "print(f\"MLP Test Accuracy: {accuracy(y_test, y_pred_test):.2f}%\")" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.12" } }, "nbformat": 4, "nbformat_minor": 5 }