Linear combination
import numpy as np
# y = w1*x1 + w2*x2 + w3*x3 + bias
x = np.array([2, 3, 5]) # features
w = np.array([0.4, 0.1, 0.7]) # weights
b = 1.2 # bias
y = np.dot(w, x) + b
# 0.4*2 + 0.1*3 + 0.7*5 + 1.2 = 5.9
Linear regression from scratch
# predict house price from size & rooms
X = np.array([[50, 2], [80, 3], [120, 4], [60, 2]]) # features
y = np.array([100, 150, 200, 110]) # prices
w = np.array([1.5, 10.0]) # weights
b = 5.0 # bias
# predictions = X @ w + b
y_pred = X @ w + b
# [85., 155., 225., 105.]
# error
residuals = y - y_pred
# [15., -5., -25., 5.]
Mean Squared Error (MSE)
mse = np.mean((y - y_pred) ** 2)
# average of squared errors — the loss we minimize
rmse = np.sqrt(mse)
# same unit as y — easier to interpret
Gradient (slope of the loss)
# how much does MSE change if we nudge each weight?
# dL/dw = -2/n * X.T @ (y - y_pred)
n = len(y)
grad_w = (-2 / n) * X.T @ (y - y_pred)
grad_b = (-2 / n) * np.sum(y - y_pred)
Gradient descent (update weights)
lr = 0.001 # learning rate
for _ in range(1000):
y_pred = X @ w + b
grad_w = (-2 / n) * X.T @ (y - y_pred)
grad_b = (-2 / n) * np.sum(y - y_pred)
w -= lr * grad_w # step opposite to gradient
b -= lr * grad_b
print(w, b) # converged weights
Dot product as similarity
# how similar are two vectors?
a = np.array([1, 0, 1]) # user A preferences
b = np.array([1, 1, 0]) # user B preferences
np.dot(a, b) # 1 — shared interest count
# used in recommendations, NLP, search
Weighted average
scores = np.array([80, 90, 70])
weights = np.array([0.2, 0.5, 0.3]) # must sum to 1
np.dot(scores, weights) # 82.0
# same as a linear combination where weights sum to 1
Standardization (z-score)
x = np.array([10, 20, 30, 40, 50])
z = (x - x.mean()) / x.std()
# [-1.41, -0.71, 0., 0.71, 1.41]
# mean=0, std=1 — models train way better on this
Min-max normalization
x = np.array([10, 20, 30, 40, 50])
norm = (x - x.min()) / (x.max() - x.min())
# [0., 0.25, 0.5, 0.75, 1.]
# squashes everything into [0, 1]
Variance & covariance
x = np.array([2, 4, 4, 6])
y = np.array([1, 3, 2, 5])
np.var(x) # 2.0 — spread of x
np.cov(x, y)
# [[2.67, 2.33],
# [2.33, 2.92]]
# diagonal = variances, off-diagonal = covariance
# positive covariance = they move together
Correlation
np.corrcoef(x, y)
# [[1. , 0.88],
# [0.88, 1. ]]
# 1.0 = perfect correlation
# 0.0 = no relationship
# -1.0 = perfect inverse
Sigmoid activation
# squashes linear combo output into probability
def sigmoid(z):
return 1 / (1 + np.exp(-z))
z = np.dot(w, x) + b # linear combination first
prob = sigmoid(z) # then squeeze into (0, 1)
# this is literally logistic regression