optim
¶
Optimizer (ABC)
¶
Base class for FunFact optimizers.
To add your own optimizer:
- inherit from
Optimizer
- implement two methods: an initializer
__init__
to set the model and optimizer parameters, and astep
method to take a step based on the gradient.
Source code in funfact/optim.py
class Optimizer(ABC):
'''Base class for FunFact optimizers.
To add your own optimizer:
- *inherit* from `Optimizer`
- implement two methods: an initializer `__init__` to set
the model and optimizer parameters, and a `step` method to take a step
based on the gradient.'''
@abstractmethod
def __init__(self, model, **kwargs):
'''Initialize optimizer.
Args:
model (factors): factors from factorization model to be optimized.
kwargs: (hyper)parameters for optimizer.
'''
@abstractmethod
def step(self, grad):
'''Take a step in the optimization process.
Args:
grad (factors): gradiemts of the factors
'''
Adam (Optimizer)
¶
Source code in funfact/optim.py
class Adam(Optimizer):
def __init__(
self, X, lr=0.1, beta1=0.9, beta2=0.999, epsilon=1e-7, **kwargs
):
'''Adam gradient descent optimizer.
Args:
X (factors): factors from factorization model to be optimized.
lr (float): learning rate.
beta1 (float): first order moment.
beta2 (float): second order moment.
epsilon (float): perturbation to improve numerical stablility.
'''
self.X = X
self.lr = lr
self.beta1 = beta1
self.beta2 = beta2
self.epsilon = epsilon
self.M = [ab.zeros_like(x) for x in X]
self.V = [ab.zeros_like(x) for x in X]
def step(self, grad):
for i, g in enumerate(grad):
self.M[i] = self.beta1 * self.M[i] + (1 - self.beta1) * g
self.V[i] = self.beta2 * self.V[i] + \
(1 - self.beta2) * g * ab.conj(g)
mhat = self.M[i] / (1 - self.beta1)
vhat = self.V[i] / (1 - self.beta2)
self.X[i] -= self.lr * mhat * ab.reciprocal(ab.sqrt(
vhat + self.epsilon
))
__init__(self, X, lr=0.1, beta1=0.9, beta2=0.999, epsilon=1e-07, **kwargs)
special
¶
Adam gradient descent optimizer.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
X |
factors |
factors from factorization model to be optimized. |
required |
lr |
float |
learning rate. |
0.1 |
beta1 |
float |
first order moment. |
0.9 |
beta2 |
float |
second order moment. |
0.999 |
epsilon |
float |
perturbation to improve numerical stablility. |
1e-07 |
Source code in funfact/optim.py
def __init__(
self, X, lr=0.1, beta1=0.9, beta2=0.999, epsilon=1e-7, **kwargs
):
'''Adam gradient descent optimizer.
Args:
X (factors): factors from factorization model to be optimized.
lr (float): learning rate.
beta1 (float): first order moment.
beta2 (float): second order moment.
epsilon (float): perturbation to improve numerical stablility.
'''
self.X = X
self.lr = lr
self.beta1 = beta1
self.beta2 = beta2
self.epsilon = epsilon
self.M = [ab.zeros_like(x) for x in X]
self.V = [ab.zeros_like(x) for x in X]
step(self, grad)
¶
Take a step in the optimization process.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
grad |
factors |
gradiemts of the factors |
required |
Source code in funfact/optim.py
def step(self, grad):
for i, g in enumerate(grad):
self.M[i] = self.beta1 * self.M[i] + (1 - self.beta1) * g
self.V[i] = self.beta2 * self.V[i] + \
(1 - self.beta2) * g * ab.conj(g)
mhat = self.M[i] / (1 - self.beta1)
vhat = self.V[i] / (1 - self.beta2)
self.X[i] -= self.lr * mhat * ab.reciprocal(ab.sqrt(
vhat + self.epsilon
))
RMSprop (Optimizer)
¶
Source code in funfact/optim.py
class RMSprop(Optimizer):
def __init__(
self, X, lr=0.1, alpha=0.99, epsilon=1e-8, weight_decay=0,
momentum=0, centered=False, **kwargs
):
'''RMSprop gradient descent optimizer with momentum.
Args:
X (factors): factors from factorization model to be optimized.
lr (float): learning rate.
alpha (float): smoothing constant.
epsilon (float): perturbation to improve numerical stablility.
weight_decay (float): weight decay.
momentum (float): momentum factor.
centered (bool): centered RMPSprop.
'''
self.X = X
self.lr = lr
self.alpha = alpha
self.epsilon = epsilon
self.weight_decay = weight_decay
self.momentum = momentum
self.centered = centered
self.V = [ab.zeros_like(x) for x in X]
self.B = [ab.zeros_like(x) for x in X]
self.G = [ab.zeros_like(x) for x in X]
def step(self, grad):
if self.weight_decay != 0:
for g, x in zip(grad, self.X):
g += self.weight_decay * x
for i, g in enumerate(grad):
self.V[i] = self.alpha * self.V[i] + \
(1 - self.alpha) * g * ab.conj(g)
vhat = self.V[i] / (1 - self.alpha)
if self.centered:
self.G[i] = self.alpha * self.G[i] + \
(1 - self.alpha) * g
vhat -= self.G[i] * self.G[i]
if self.momentum > 0:
self.B[i] = self.momentum * self.B[i] + \
g * ab.reciprocal(
ab.sqrt(vhat) + self.epsilon)
self.X[i] -= self.lr * self.B[i]
else:
self.X[i] -= self.lr * g * ab.reciprocal(
ab.sqrt(vhat + self.epsilon))
__init__(self, X, lr=0.1, alpha=0.99, epsilon=1e-08, weight_decay=0, momentum=0, centered=False, **kwargs)
special
¶
RMSprop gradient descent optimizer with momentum.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
X |
factors |
factors from factorization model to be optimized. |
required |
lr |
float |
learning rate. |
0.1 |
alpha |
float |
smoothing constant. |
0.99 |
epsilon |
float |
perturbation to improve numerical stablility. |
1e-08 |
weight_decay |
float |
weight decay. |
0 |
momentum |
float |
momentum factor. |
0 |
centered |
bool |
centered RMPSprop. |
False |
Source code in funfact/optim.py
def __init__(
self, X, lr=0.1, alpha=0.99, epsilon=1e-8, weight_decay=0,
momentum=0, centered=False, **kwargs
):
'''RMSprop gradient descent optimizer with momentum.
Args:
X (factors): factors from factorization model to be optimized.
lr (float): learning rate.
alpha (float): smoothing constant.
epsilon (float): perturbation to improve numerical stablility.
weight_decay (float): weight decay.
momentum (float): momentum factor.
centered (bool): centered RMPSprop.
'''
self.X = X
self.lr = lr
self.alpha = alpha
self.epsilon = epsilon
self.weight_decay = weight_decay
self.momentum = momentum
self.centered = centered
self.V = [ab.zeros_like(x) for x in X]
self.B = [ab.zeros_like(x) for x in X]
self.G = [ab.zeros_like(x) for x in X]
step(self, grad)
¶
Take a step in the optimization process.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
grad |
factors |
gradiemts of the factors |
required |
Source code in funfact/optim.py
def step(self, grad):
if self.weight_decay != 0:
for g, x in zip(grad, self.X):
g += self.weight_decay * x
for i, g in enumerate(grad):
self.V[i] = self.alpha * self.V[i] + \
(1 - self.alpha) * g * ab.conj(g)
vhat = self.V[i] / (1 - self.alpha)
if self.centered:
self.G[i] = self.alpha * self.G[i] + \
(1 - self.alpha) * g
vhat -= self.G[i] * self.G[i]
if self.momentum > 0:
self.B[i] = self.momentum * self.B[i] + \
g * ab.reciprocal(
ab.sqrt(vhat) + self.epsilon)
self.X[i] -= self.lr * self.B[i]
else:
self.X[i] -= self.lr * g * ab.reciprocal(
ab.sqrt(vhat + self.epsilon))