优化器 ¶

优化器根据网络反向传播的梯度信息来更新网络的参数，以起到降低 loss 函数计算值，使得模型输出更加接近真实标签。

PyTorch 很人性化的给我们提供了一个优化器的库 torch.optim，在这里面提供了多种优化器，包括基类 torch.optim.Optimizer，如下（有删改）：

In [ ]:

Copied!





# my optimizer
from collections import defaultdict
class MyOptimizer:
    r"""Base class for all optimizers.
    Args:
        params (iterable): an iterable of :class:`torch.Tensor` s or
            :class:`dict` s. Specifies what Tensors should be optimized.
        defaults: (dict): a dict containing default values of optimization
            options (used when a parameter group doesn't specify them).
    """
    def __init__(self, params, defaults):        
        self.defaults = defaults # hyperparameters
        self.state = defaultdict(dict) # cache of paremeters
        self.param_groups = list(params)
    
    def zero_grad(self, set_to_none: bool = True) -> None:
        """
        Clear the parameter gradients.
        """
        pass

    def step(self, closure: None = ...) -> None:
        """
        Performs a single optimization step.
        """
        pass
    def add_param_group(self, param_group: dict) -> None:
        """
        Add a param group to the :class:`Optimizer` s `param_groups`.
        
        This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the
        :class:`Optimizer` as training progresses.
        """
        pass

myoptimizer = MyOptimizer(model.parameters(), lr=0.01)
lossf = MyLossFunction()
for epoch in range(EPOCH):
    ...
    myoptimizer.zero_grad()
    loss = lossf(...)
    loss.backward()
    myoptimizer.step()
# my optimizer
from collections import defaultdict
class MyOptimizer:
    r"""Base class for all optimizers.
    Args:
        params (iterable): an iterable of :class:`torch.Tensor` s or
            :class:`dict` s. Specifies what Tensors should be optimized.
        defaults: (dict): a dict containing default values of optimization
            options (used when a parameter group doesn't specify them).
    """
    def __init__(self, params, defaults):        
        self.defaults = defaults # hyperparameters
        self.state = defaultdict(dict) # cache of paremeters
        self.param_groups = list(params)
    
    def zero_grad(self, set_to_none: bool = True) -> None:
        """
        Clear the parameter gradients.
        """
        pass

    def step(self, closure: None = ...) -> None:
        """
        Performs a single optimization step.
        """
        pass
    def add_param_group(self, param_group: dict) -> None:
        """
        Add a param group to the :class:`Optimizer` s `param_groups`.
        
        This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the
        :class:`Optimizer` as training progresses.
        """
        pass

myoptimizer = MyOptimizer(model.parameters(), lr=0.01)
lossf = MyLossFunction()
for epoch in range(EPOCH):
    ...
    myoptimizer.zero_grad()
    loss = lossf(...)
    loss.backward()
    myoptimizer.step()