# %% [markdown]
# # 3. Explore the steps involved in the machine learning workflow and explain it using appropriate libraries and methods

# %%
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder, StandardScaler
from sklearn.feature_selection import mutual_info_classif, mutual_info_regression

# %% [markdown]
# a. Download any one of the following datasets:
#     
# Loan Prediction Problem
# https://www.kaggle.com/datasets/altruistdelhite04/loan-prediction-problem-dataset
# 

# %% [markdown]
# b. Load the dataset into Python using the Pandas library.

# %%
df = pd.read_csv('loan-prediction-problem-dataset/train_u6lujuX_CVtuZ9i.csv')
# c. Display the first five records of the dataset.
df.head(n=5)

# %% [markdown]
# d. Determine: Number of rows, Number of columns, Feature names, Target (label) column
# 
# This is a regression task of predicting the loan amount approved, given features like approval status, incommes, gender, etc.

# %%
nrow, ncol = df.shape
print(f"Rows = {nrow}\nCols = {ncol}")

# %%
target_col = "LoanAmount"

print("Feature Names")
feats = df.columns.tolist()
feats.remove(target_col)
feats
print(f"Target column: {target_col}")

# %% [markdown]
# ### e. Data Preprocessing

# %% [markdown]
# Handling missing vlaues
# https://stackoverflow.com/questions/70402677/define-a-strategy-of-filling-nans-in-pandas-dataframe
# 

# %%
# Missing values per column
df.isna().sum()

# %%
df[df['Self_Employed'] == "Yes"].shape[0]

# %%
df[df['Self_Employed'] == "No"].shape[0]

# %%
prev_nan = df[df['Self_Employed'].isna()].index.tolist()
df[df['Self_Employed'].isna()].head(n=5)

# %% [markdown]
# For every multiclass columns, we fill the NaN, values with the most common class.

# %%
o = df.select_dtypes('str')
print(o.columns)
df[o.columns] = o.fillna(o.agg(lambda x: x.mode().values[0]))

# Checking the changes
df.iloc[prev_nan].head(5)

# %%
df.isna().sum()

# %% [markdown]
# For numerical values, I am taking the median value to fill the NaN

# %%
f = df.select_dtypes('float64')
print(f.columns)

# %%
df[f.columns] = f.fillna(f.median())
df.isna().sum()

# %% [markdown]
# Remove irrelevant features:
# Loan_ID is an irrelevent feature for the regression task, hence should be removed.

# %%
df.drop(columns=['Loan_ID'], inplace=True)
df

# %% [markdown]
# Encode categorical variables - Justify the technique used for encoding
# 

# %%
{col: df[col].unique().tolist() for col in ['Gender', 'Married', 'Dependents', 'Education',
       'Self_Employed', 'Property_Area', 'Loan_Status']}

# %% [markdown]
# Binary Encoding - Gender, Married, Education, Self_Employed, Loan_Status
# 
# Ordinal Encoding - Dependants
# 
# One hot encoding - Property_Area

# %%
# Male is 1, Female is 0
df['Gender'] = (df['Gender'] == 'Male').astype(int)
# Similarly
df['Married'] = (df['Married'] == 'Yes').astype(int)
df['Education'] = (df['Education'] == 'Graduate').astype(int)
df['Self_Employed'] = (df['Self_Employed'] == 'Yes').astype(int)
df['Loan_Status'] = (df['Loan_Status'] == 'Y').astype(int)
df

# %%
ord_enc = OrdinalEncoder(categories=[['0', '1', '2', '3+']])
df['Dependents'] = ord_enc.fit_transform(df[['Dependents']])
df['Dependents'].unique().tolist()

# %%
# Apply one-hot encoding to the 'Color' column
df = pd.get_dummies(df, columns=['Property_Area'], dtype=int)
df.sample(5)

# %% [markdown]
# Normalize or standardize numerical features

# %%
# Making a copy for later reference
old_df = df.copy()

# %%
# Log transformation for high skew cols
log_features = ['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount']
for col in log_features:
    df[f'{col}_log'] = np.log1p(df[col])
df

# %%
# Scaling for some
features_to_scale = [
    'ApplicantIncome_log', 
    'CoapplicantIncome_log', 
    'LoanAmount_log', 
    'Loan_Amount_Term'
]

scaler = StandardScaler()
df[features_to_scale] = scaler.fit_transform(df[features_to_scale])
df[log_features + features_to_scale]

# %%
df = df.drop(columns=log_features)
df

# %% [markdown]
# ### f. Compute the Correlation Coefficient and Mutual Information scores for all input features with respect to
# the target variable. Tabulate the results and identify the top ‘n’ important features.

# %%

tg = 'LoanAmount_log'
feats = df.drop(columns=[tg])
target = df[tg]

# Correlation Coefficient
corr_matrix = df.corr(numeric_only=True)
correlation_scores = corr_matrix[tg].drop(tg)

# %%
mi_score = mutual_info_regression(feats, target, random_state=42)

results_df = pd.DataFrame(
    {
        "Correlation_Coefficient": correlation_scores,
        "Mutual_Information": mi_score,
    },
    index=feats.columns,
)
results_df

# %%
# Top 5 imoportant features
results_df = results_df.sort_values(by="Mutual_Information", ascending=False)
results_df.head(5)

# %% [markdown]
# ### g. Exploratory Data Analysis (EDA) and Visualization: 
# 
# Examine the dataset using summary statistics and visualize the data using plots such as histograms, bar charts, scatter plots, box plots, heatmaps, and pair plots.

# %%
print("Dataset Information:")
old_df.info()

# %%
df.describe()

# %%
# 2. Histograms (Distribution of all numerical features)
old_df.hist(figsize=(15, 12), bins=20, edgecolor='black')
plt.suptitle('Histograms of Numerical Features', fontsize=16)
plt.show()

# %%
# 3. Bar Chart (Categorical data example: Loan_Status)
plt.figure(figsize=(6, 4))
old_df['Loan_Status'].value_counts().plot(kind='bar', color=['skyblue', 'salmon'], edgecolor='black')
plt.title('Loan Status Distribution')
plt.xlabel('Loan Status')
plt.ylabel('Count')
plt.xticks(rotation=0)
plt.show()

# %%
# 4. Scatter Plot (Relationship between ApplicantIncome and LoanAmount)
plt.figure(figsize=(8, 6))
plt.scatter(old_df['ApplicantIncome'], old_df['LoanAmount'], alpha=0.5, c='blue')
plt.title('Applicant Income vs. Loan Amount')
plt.xlabel('Applicant Income')
plt.ylabel('Loan Amount')
plt.grid(True, linestyle='--', alpha=0.6)
plt.show()

# %%
# 5. Box Plot (Outlier detection: LoanAmount grouped by Loan_Status)
old_df.boxplot(column='LoanAmount', by='Loan_Status', figsize=(8, 6), grid=False)
plt.title('Loan Amount by Loan Status')
plt.suptitle('') # Removes default pandas subtitle
plt.ylabel('Loan Amount')
plt.show()

# %%
# 6. Heatmap (Correlation matrix using Matplotlib)
plt.figure(figsize=(12, 10))
corr_matrix = df.corr()
cax = plt.imshow(corr_matrix, cmap='coolwarm', interpolation='nearest')
plt.colorbar(cax)
# Add labels
ticks = np.arange(len(corr_matrix.columns))
plt.xticks(ticks, corr_matrix.columns, rotation=90)
plt.yticks(ticks, corr_matrix.columns)
# Annotate values
for i in range(len(corr_matrix.columns)):
    for j in range(len(corr_matrix.columns)):
        plt.text(j, i, f"{corr_matrix.iloc[i, j]:.2f}", ha='center', va='center', color='black', fontsize=8)
plt.title('Correlation Heatmap', pad=20)
plt.show()

# %%
# 7. Pair Plot (Scatter Matrix for selected continuous variables)
# Selecting a subset to ensure readability
continuous_vars = ['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount', 'Loan_Amount_Term']
pd.plotting.scatter_matrix(old_df[continuous_vars], figsize=(12, 12), marker='o', alpha=0.6, hist_kwds={'bins': 20})
plt.suptitle('Pair Plot (Scatter Matrix)', fontsize=16)
plt.show()

# %% [markdown]
# ### h. Based on the visualizations, answer the following:
# 1. Which features appear normally distributed?
# 2. Is the dataset balanced or imbalanced (for classification datasets using any chart)?

# %% [markdown]
# Answers: 
# 1. None of the features exhibit a strict normal distribution. LoanAmount provides the closest approximation, displaying a general bell-shaped curve, though it contains observable positive skewness and right-tail outliers. The other continuous variables, ApplicantIncome and CoapplicantIncome, are heavily right-skewed. The remaining variables are discrete or categorical.
# 2. This is a regression task

# %% [markdown]
# ### i. Data Splitting
# Divide the dataset into training, validation, and testing sets for model development and evaluation.

# %%
from sklearn.model_selection import train_test_split

df_train, df_rem = train_test_split(df, train_size=0.70, random_state=42)
df_val, df_test = train_test_split(df_rem, test_size=0.50, random_state=42)
print(len(df_train), len(df_val), len(df_test))