NumPy & Pandas for Machine Learning

Real World Data (CSV/SQL/Excel)
         ↓  Pandas  (explore, clean, transform)
Clean DataFrame
         ↓  .values or .to_numpy()
NumPy Array
         ↓  scikit-learn / TensorFlow / PyTorch
Trained Model

import numpy as np
import time

# Python list approach
python_list = list(range(1_000_000))
start = time.time()
result = [x * 2 for x in python_list]
print(f"Python list: {time.time() - start:.3f}s")   # ~0.100s

# NumPy approach
numpy_array = np.arange(1_000_000)
start = time.time()
result = numpy_array * 2                             # No loop needed!
print(f"NumPy array: {time.time() - start:.3f}s")   # ~0.002s — 50x faster

import numpy as np

# --- From Python data ---
arr1 = np.array([1, 2, 3, 4, 5])                    # 1D array, shape (5,)
arr2 = np.array([[1, 2, 3], [4, 5, 6]])             # 2D array, shape (2, 3)

# --- Ranges ---
arr3 = np.arange(0, 10, 2)                          # [0, 2, 4, 6, 8]
arr4 = np.linspace(0, 1, 5)                         # [0.0, 0.25, 0.5, 0.75, 1.0]

# --- Pre-filled ---
zeros  = np.zeros((3, 4))                            # 3 rows, 4 cols of 0.0
ones   = np.ones((2, 2))                             # [[1., 1.], [1., 1.]]
eye    = np.eye(3)                                   # 3×3 identity matrix
rand   = np.random.rand(3, 3)                        # uniform random [0, 1)
randn  = np.random.randn(100)                        # 100 standard-normal samples

# --- Shape inspection ---
print(arr2.shape)   # (2, 3)  ← rows, cols
print(arr2.dtype)   # int64
print(arr2.size)    # 6  ← total elements
print(arr2.ndim)    # 2  ← number of dimensions

arr = np.array([[10, 20, 30],
                [40, 50, 60],
                [70, 80, 90]])

# Basic indexing (row, col)
print(arr[0, 2])       # 30  — first row, third column
print(arr[-1, -1])     # 90  — last row, last column

# Slicing (like Python but in 2D)
print(arr[0, :])       # [10, 20, 30]  — entire first row
print(arr[:, 1])       # [20, 50, 80]  — entire second column
print(arr[0:2, 1:3])   # [[20, 30], [50, 60]] — top-right 2×2 block

# Boolean indexing — the ML workhorse
mask = arr > 40
print(arr[mask])       # [50, 60, 70, 80, 90]  — all values > 40

# Fancy indexing — select specific rows
rows_wanted = [0, 2]
print(arr[rows_wanted])  # [[10, 20, 30], [70, 80, 90]]

a = np.array([1, 2, 3, 4])
b = np.array([10, 20, 30, 40])

# Element-wise operations (no loops needed!)
print(a + b)            # [11, 22, 33, 44]
print(a * b)            # [10, 40, 90, 160]
print(a ** 2)           # [1, 4, 9, 16]
print(np.sqrt(a))       # [1., 1.41, 1.73, 2.]
print(np.log(a))        # natural log of each element

# Dot product (critical for ML!)
print(np.dot(a, b))     # 1×10 + 2×20 + 3×30 + 4×40 = 300
# Or equivalently:
print(a @ b)            # 300 (@ is matrix multiplication operator)

# Aggregations
print(np.sum(a))        # 10
print(np.mean(a))       # 2.5
print(np.std(a))        # standard deviation
print(np.max(a), np.min(a))   # 4, 1

# Axis-wise aggregations on 2D arrays
matrix = np.array([[1, 2, 3],
                   [4, 5, 6]])
print(np.sum(matrix, axis=0))  # [5, 7, 9] — sum down columns
print(np.sum(matrix, axis=1))  # [6, 15]   — sum across rows

arr = np.arange(12)          # [0, 1, 2, ..., 11]

# Reshape to 2D
m = arr.reshape(3, 4)        # 3 rows, 4 cols
m = arr.reshape(4, 3)        # 4 rows, 3 cols
m = arr.reshape(2, 2, 3)     # 3D — 2 groups of 2×3 matrices

# -1 means "figure out this dimension automatically"
m = arr.reshape(-1, 3)       # (4, 3) — NumPy computes 4 for you
m = arr.reshape(3, -1)       # (3, 4)

# Flatten — turn any shape back to 1D
print(m.flatten())           # [0, 1, 2, ..., 11]

# Transposing (swap rows and columns)
m = np.array([[1, 2, 3], [4, 5, 6]])
print(m.T)                   # [[1, 4], [2, 5], [3, 6]] — shape (3, 2)

# Normalize a dataset: subtract mean, divide by std
# Without broadcasting, you'd need a loop
data = np.array([[1.0, 2.0, 3.0],
                 [4.0, 5.0, 6.0],
                 [7.0, 8.0, 9.0]])   # shape (3, 3)

column_means = np.mean(data, axis=0)   # shape (3,) — one mean per column
column_stds  = np.std(data, axis=0)    # shape (3,)

# Broadcasting: (3,3) - (3,) → NumPy "broadcasts" (3,) to (3,3) automatically!
normalized = (data - column_means) / column_stds
# This is Z-score normalization — used EVERYWHERE in ML preprocessing!

print(normalized)
# Each column now has mean ≈ 0 and std ≈ 1

import numpy as np

# ----- Cosine Similarity (used in RAG, NLP, recommendation systems) -----
def cosine_similarity(vec_a: np.ndarray, vec_b: np.ndarray) -> float:
    """Measure the angle between two vectors. 1.0 = identical, 0.0 = perpendicular."""
    dot_product = np.dot(vec_a, vec_b)
    norm_a = np.linalg.norm(vec_a)    # ||a|| — Euclidean length
    norm_b = np.linalg.norm(vec_b)
    return dot_product / (norm_a * norm_b)

v1 = np.array([1.0, 2.0, 3.0])
v2 = np.array([1.0, 2.0, 3.0])   # identical
v3 = np.array([3.0, -1.0, 0.0])  # different

print(f"Identical vectors: {cosine_similarity(v1, v2):.4f}")   # 1.0000
print(f"Different vectors: {cosine_similarity(v1, v3):.4f}")   # 0.2673

# ----- Softmax (used in neural network output layers) -----
def softmax(x: np.ndarray) -> np.ndarray:
    """Convert raw scores to probabilities that sum to 1."""
    # Subtract max for numerical stability (prevents overflow)
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum()

logits = np.array([2.0, 1.0, 0.1])
probs = softmax(logits)
print(f"Probabilities: {probs}")         # [0.659, 0.242, 0.099]
print(f"Sum: {probs.sum():.4f}")         # 1.0000

# ----- Mean Squared Error Loss -----
def mse_loss(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """The loss function linear regression minimizes."""
    return np.mean((y_true - y_pred) ** 2)

y_true = np.array([3.0, -0.5, 2.0, 7.0])
y_pred = np.array([2.5,  0.0, 2.0, 8.0])
print(f"MSE: {mse_loss(y_true, y_pred):.4f}")   # 0.3750

import pandas as pd

# Every ML project starts with one of these:
df = pd.read_csv("data.csv")              # CSV files
df = pd.read_excel("data.xlsx")          # Excel files
df = pd.read_json("data.json")           # JSON
df = pd.read_sql("SELECT * FROM users", conn)  # SQL databases
df = pd.read_parquet("data.parquet")    # Parquet (big data)

import pandas as pd

# Creating a DataFrame manually (great for experiments)
data = {
    "name":   ["Alice", "Bob", "Carol", "Dave", "Eve"],
    "age":    [25, 30, 35, None, 28],         # None = missing value
    "salary": [50000, 75000, 90000, 65000, 80000],
    "dept":   ["Eng", "Eng", "HR", "Finance", "Eng"],
    "joined": ["2020-01-15", "2019-03-22", "2018-07-01", "2021-11-10", "2022-05-20"],
}
df = pd.DataFrame(data)

# ===== First 5 commands every data scientist runs =====
print(df.head())            # first 5 rows — visual check
print(df.tail(3))           # last 3 rows
print(df.shape)             # (5, 5) — rows × columns
print(df.dtypes)            # data type of each column
print(df.info())            # concise summary (non-nulls, dtypes, memory)
print(df.describe())        # count, mean, std, min, 25%, 50%, 75%, max for numerics

# loc  → label-based  (use column NAMES, row LABELS)
# iloc → integer-based (use row/col NUMBERS, 0-indexed)

# --- loc examples ---
print(df.loc[0])                       # row with index label 0
print(df.loc[0:2])                     # rows 0, 1, 2 (INCLUSIVE on both ends!)
print(df.loc[0, "name"])               # "Alice" — row 0, column "name"
print(df.loc[:, ["name", "salary"]])   # all rows, specific columns
print(df.loc[df["dept"] == "Eng"])     # boolean filter — only Eng rows

# --- iloc examples ---
print(df.iloc[0])                      # first row (by position)
print(df.iloc[0:2])                    # rows 0, 1 (exclusive on right, like Python!)
print(df.iloc[0, 1])                   # row 0, column 1 → 25 (age)
print(df.iloc[:, [0, 2]])              # all rows, columns at positions 0 and 2

# --- Shorthand column access ---
print(df["name"])                      # Series — one column
print(df[["name", "salary"]])          # DataFrame — multiple columns
print(df.name)                         # Same as df["name"] — works for simple names

# Inspect missing data
print(df.isnull())           # True/False for each cell
print(df.isnull().sum())     # count of missing per column
print(df.isnull().sum() / len(df) * 100)  # % missing per column

# Strategy 1: Drop rows with any missing value
df_clean = df.dropna()

# Strategy 2: Drop columns with too many missing values
df_clean = df.dropna(axis=1, thresh=4)   # keep col only if 4+ non-null values

# Strategy 3: Fill with a fixed value
df["age"] = df["age"].fillna(0)

# Strategy 4: Fill with mean (most common for ML!)
df["age"] = df["age"].fillna(df["age"].mean())

# Strategy 5: Fill with median (better for skewed data)
df["age"] = df["age"].fillna(df["age"].median())

# Strategy 6: Forward fill (great for time-series)
df["age"] = df["age"].fillna(method="ffill")

# Boolean filters (the main way)
eng_staff = df[df["dept"] == "Eng"]
high_earners = df[df["salary"] > 70000]
young_eng = df[(df["dept"] == "Eng") & (df["age"] < 30)]   # & for AND
senior_or_rich = df[(df["age"] > 32) | (df["salary"] > 80000)]  # | for OR

# .query() — more readable for complex conditions
high_earners = df.query("salary > 70000 and dept == 'Eng'")

# .isin() — filter by list of values
tech_depts = df[df["dept"].isin(["Eng", "Finance"])]

# .between() — range filter
mid_earners = df[df["salary"].between(60000, 80000)]

# Add new columns (feature engineering — critical for ML!)
df["salary_k"] = df["salary"] / 1000                    # salary in thousands
df["senior"] = df["age"] > 30                           # boolean feature
df["age_squared"] = df["age"] ** 2                      # polynomial feature

# Apply a function to a column
df["name_length"] = df["name"].apply(len)
df["salary_band"] = df["salary"].apply(
    lambda x: "high" if x > 80000 else "mid" if x > 60000 else "low"
)

# Apply to multiple columns at once
df["annual_tax"] = df.apply(
    lambda row: row["salary"] * 0.3 if row["age"] > 28 else row["salary"] * 0.2,
    axis=1   # axis=1 means "apply row-by-row"
)

# String operations on text columns
df["dept_lower"] = df["dept"].str.lower()
df["name_upper"] = df["name"].str.upper()
df["has_a"] = df["name"].str.contains("a", case=False)

# groupby is the SQL GROUP BY equivalent
dept_stats = df.groupby("dept")["salary"].mean()
print(dept_stats)
# dept
# Eng        68333.333
# Finance    65000.000
# HR         90000.000

# Multiple aggregations at once
dept_summary = df.groupby("dept").agg(
    avg_salary  = ("salary", "mean"),
    max_salary  = ("salary", "max"),
    headcount   = ("name", "count"),
    avg_age     = ("age", "mean")
)
print(dept_summary)

# value_counts — most useful one-liner in Pandas
print(df["dept"].value_counts())          # count of each department
print(df["dept"].value_counts(normalize=True))  # as percentages

# One-Hot Encoding: "dept" → separate 0/1 columns per category
# This is what you MUST do before passing data to sklearn
df_encoded = pd.get_dummies(df, columns=["dept"], prefix="dept")
print(df_encoded.columns.tolist())
# ['name', 'age', 'salary', 'joined', 'dept_Eng', 'dept_Finance', 'dept_HR']
# Each dept_ column is 0 or 1

# Label Encoding: "dept" → integer (for tree models)
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
df["dept_code"] = le.fit_transform(df["dept"])
# Eng → 0, Finance → 1, HR → 2

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# ===== Step 1: Load =====
df = pd.read_csv("employees.csv")
print(f"Loaded: {df.shape[0]} rows, {df.shape[1]} columns")
print(df.isnull().sum())

# ===== Step 2: Explore =====
print(df.describe())
print(df["dept"].value_counts())

# ===== Step 3: Clean =====
df["age"] = df["age"].fillna(df["age"].median())         # fill missing age
df = df.dropna(subset=["salary"])                        # drop if salary missing

# ===== Step 4: Feature Engineering =====
df["salary_log"] = np.log1p(df["salary"])               # log transform (reduces skew)
df["seniority"] = df["age"] - 22                        # years since typical grad age
df["joined"] = pd.to_datetime(df["joined"])             # parse dates
df["tenure_days"] = (pd.Timestamp.now() - df["joined"]).dt.days

# ===== Step 5: Encode Categoricals =====
df = pd.get_dummies(df, columns=["dept"], drop_first=True)  # drop_first avoids dummy trap

# ===== Step 6: Define Features and Target =====
target = "high_performer"     # assume this column is our binary target
feature_cols = [c for c in df.columns if c not in [target, "name", "joined"]]
X = df[feature_cols].values   # → NumPy array  (use .to_numpy() alternatively)
y = df[target].values         # → NumPy array

# ===== Step 7: Split =====
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# ===== Step 8: Scale (AFTER splitting — never before!) =====
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)    # fit on train only
X_test  = scaler.transform(X_test)         # apply same transform to test

print(f"Train: {X_train.shape}, Test: {X_test.shape}")
# Your data is now ready for any sklearn model

Feature             Python List         NumPy Array
------              -----------         -----------
Types               Mixed OK            Single type only
Speed               Slow (interpreted)  Fast (C-level, vectorized)
Memory              More (pointers)     Less (contiguous)
Multi-dimensional   Nested lists        Native n-dimensional
Math operations     Manual loops        Element-wise automatic
ML compatibility    ✗ (not accepted)    ✓ (standard input)

Use Python lists for:  Heterogeneous data, small collections, appending
Use NumPy arrays for:  Numerical computation, ML features, matrix ops

df.loc[row_label, col_name]     # Labels  — df.loc[0, "name"]   — right boundary INCLUDED
df.iloc[row_pos,  col_pos]      # Integers — df.iloc[0, 1]      — right boundary EXCLUDED

Mental model:
loc  → dictionary-style lookup by name
iloc → list-style lookup by position

# ❌ WRONG — data leakage!
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)           # uses ALL data including test
X_train, X_test = train_test_split(X_scaled)

# ✓ CORRECT — fit only on training data
X_train, X_test = train_test_split(X)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)      # fit AND transform train
X_test  = scaler.transform(X_test)           # ONLY transform test (no fit!)