# -*- coding: utf-8 -*-
from setuptools import setup

packages = \
['pytorch_optimizer']

package_data = \
{'': ['*']}

install_requires = \
['numpy>=1.21.1,<2.0.0', 'torch>=1.11.0,<2.0.0']

setup_kwargs = {
    'name': 'pytorch-optimizer',
    'version': '1.1.1',
    'description': 'Bunch of optimizer implementations in PyTorch with clean-code, strict types. Also, including useful optimization ideas.',
    'long_description': 'pytorch-optimizer\n=================\n\n| |workflow| |codecov| |Documentation Status| |PyPI version| |PyPi download| |black|\n\n| Bunch of optimizer implementations in PyTorch with clean-code, strict types. Also, including useful optimization ideas.\n| Most of the implementations are based on the original paper, but I added some tweaks.\n| Highly inspired by `pytorch-optimizer <https://github.com/jettify/pytorch-optimizer>`__.\n\nDocumentation\n-------------\n\nhttps://pytorch-optimizers.readthedocs.io/en/latest/\n\nUsage\n-----\n\nInstall\n~~~~~~~\n\n::\n\n    $ pip3 install pytorch-optimizer\n\nSimple Usage\n~~~~~~~~~~~~\n\n::\n\n    from pytorch_optimizer import AdamP\n\n    ...\n    model = YourModel()\n    optimizer = AdamP(model.parameters())\n    ...\n\nor you can use optimizer loader, simply passing a name of the optimizer.\n\n::\n\n    from pytorch_optimizer import load_optimizer\n\n    ...\n    model = YourModel()\n    opt = load_optimizer(optimizer=\'adamp\')\n    optimizer = opt(model.parameters())\n    ...\n\nSupported Optimizers\n--------------------\n\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| Optimizer    | Description                                                                            | Official Code                                                                     | Paper                                                                                         |\n+==============+========================================================================================+===================================================================================+===============================================================================================+\n| AdaBelief    | *Adapting Stepsizes by the Belief in Observed Gradients*                               | `github <https://github.com/juntang-zhuang/Adabelief-Optimizer>`__                | `https://arxiv.org/abs/2010.07468 <https://arxiv.org/abs/2010.07468>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| AdaBound     | *Adaptive Gradient Methods with Dynamic Bound of Learning Rate*                        | `github <https://github.com/Luolc/AdaBound/blob/master/adabound/adabound.py>`__   | `https://openreview.net/forum?id=Bkg3g2R9FX <https://openreview.net/forum?id=Bkg3g2R9FX>`__   |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| AdaHessian   | *An Adaptive Second Order Optimizer for Machine Learning*                              | `github <https://github.com/amirgholami/adahessian>`__                            | `https://arxiv.org/abs/2006.00719 <https://arxiv.org/abs/2006.00719>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| AdamD        | *Improved bias-correction in Adam*                                                     |                                                                                   | `https://arxiv.org/abs/2110.10828 <https://arxiv.org/abs/2110.10828>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| AdamP        | *Slowing Down the Slowdown for Momentum Optimizers on Scale-invariant Weights*         | `github <https://github.com/clovaai/AdamP>`__                                     | `https://arxiv.org/abs/2006.08217 <https://arxiv.org/abs/2006.08217>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| diffGrad     | *An Optimization Method for Convolutional Neural Networks*                             | `github <https://github.com/shivram1987/diffGrad>`__                              | `https://arxiv.org/abs/1909.11015v3 <https://arxiv.org/abs/1909.11015v3>`__                   |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| MADGRAD      | *A Momentumized, Adaptive, Dual Averaged Gradient Method for Stochastic*               | `github <https://github.com/facebookresearch/madgrad>`__                          | `https://arxiv.org/abs/2101.11075 <https://arxiv.org/abs/2101.11075>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| RAdam        | *On the Variance of the Adaptive Learning Rate and Beyond*                             | `github <https://github.com/LiyuanLucasLiu/RAdam>`__                              | `https://arxiv.org/abs/1908.03265 <https://arxiv.org/abs/1908.03265>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| Ranger       | *a synergistic optimizer combining RAdam and LookAhead, and now GC in one optimizer*   | `github <https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer>`__          | `https://bit.ly/3zyspC3 <https://bit.ly/3zyspC3>`__                                           |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| Ranger21     | *a synergistic deep learning optimizer*                                                | `github <https://github.com/lessw2020/Ranger21>`__                                | `https://arxiv.org/abs/2106.13731 <https://arxiv.org/abs/2106.13731>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| Lamb         | *Large Batch Optimization for Deep Learning*                                           | `github <https://github.com/cybertronai/pytorch-lamb>`__                          | `https://arxiv.org/abs/1904.00962 <https://arxiv.org/abs/1904.00962>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| Shampoo      | *Preconditioned Stochastic Tensor Optimization*                                        | `github <https://github.com/moskomule/shampoo.pytorch>`__                         | `https://arxiv.org/abs/1802.09568 <https://arxiv.org/abs/1802.09568>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n| Nero         | *Learning by Turning: Neural Architecture Aware Optimisation*                          | `github <https://github.com/jxbz/nero>`__                                         | `https://arxiv.org/abs/2102.07227 <https://arxiv.org/abs/2102.07227>`__                       |\n+--------------+----------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------+\n\nUseful Resources\n----------------\n\nSeveral optimization ideas to regularize & stabilize the training. Most\nof the ideas are applied in ``Ranger21`` optimizer.\n\nAlso, most of the captures are taken from ``Ranger21`` paper.\n\n+------------------------------------------+---------------------------------------------+--------------------------------------------+\n| `Adaptive Gradient Clipping`_            | `Gradient Centralization`_                  | `Softplus Transformation`_                 |\n+------------------------------------------+---------------------------------------------+--------------------------------------------+\n| `Gradient Normalization`_                | `Norm Loss`_                                | `Positive-Negative Momentum`_              |\n+------------------------------------------+---------------------------------------------+--------------------------------------------+\n| `Linear learning rate warmup`_           | `Stable weight decay`_                      | `Explore-exploit learning rate schedule`_  |\n+------------------------------------------+---------------------------------------------+--------------------------------------------+\n| `Lookahead`_                             | `Chebyshev learning rate schedule`_         | `(Adaptive) Sharpness-Aware Minimization`_ |\n+------------------------------------------+---------------------------------------------+--------------------------------------------+\n| `On the Convergence of Adam and Beyond`_ | `Gradient Surgery for Multi-Task Learning`_ |                                            |\n+------------------------------------------+---------------------------------------------+--------------------------------------------+\n\nAdaptive Gradient Clipping\n--------------------------\n\n| This idea originally proposed in ``NFNet (Normalized-Free Network)`` paper.\n| ``AGC (Adaptive Gradient Clipping)`` clips gradients based on the ``unit-wise ratio of gradient norms to parameter norms``.\n\n-  code : `github <https://github.com/deepmind/deepmind-research/tree/master/nfnets>`__\n-  paper : `arXiv <https://arxiv.org/abs/2102.06171>`__\n\nGradient Centralization\n-----------------------\n\n+-----------------------------------------------------------------------------------------------------------------+\n| .. image:: https://raw.githubusercontent.com/kozistr/pytorch_optimizer/main/assets/gradient_centralization.png  |\n+-----------------------------------------------------------------------------------------------------------------+\n\n``Gradient Centralization (GC)`` operates directly on gradients by centralizing the gradient to have zero mean.\n\n-  code : `github <https://github.com/Yonghongwei/Gradient-Centralization>`__\n-  paper : `arXiv <https://arxiv.org/abs/2004.01461>`__\n\nSoftplus Transformation\n-----------------------\n\nBy running the final variance denom through the softplus function, it lifts extremely tiny values to keep them viable.\n\n-  paper : `arXiv <https://arxiv.org/abs/1908.00700>`__\n\nGradient Normalization\n----------------------\n\nNorm Loss\n---------\n\n+---------------------------------------------------------------------------------------------------+\n| .. image:: https://raw.githubusercontent.com/kozistr/pytorch_optimizer/main/assets/norm_loss.png  |\n+---------------------------------------------------------------------------------------------------+\n\n-  paper : `arXiv <https://arxiv.org/abs/2103.06583>`__\n\nPositive-Negative Momentum\n--------------------------\n\n+--------------------------------------------------------------------------------------------------------------------+\n| .. image:: https://raw.githubusercontent.com/kozistr/pytorch_optimizer/main/assets/positive_negative_momentum.png  |\n+--------------------------------------------------------------------------------------------------------------------+\n\n-  code : `github <https://github.com/zeke-xie/Positive-Negative-Momentum>`__\n-  paper : `arXiv <https://arxiv.org/abs/2103.17182>`__\n\nLinear learning rate warmup\n---------------------------\n\n+----------------------------------------------------------------------------------------------------------+\n| .. image:: https://raw.githubusercontent.com/kozistr/pytorch_optimizer/main/assets/linear_lr_warmup.png  |\n+----------------------------------------------------------------------------------------------------------+\n\n-  paper : `arXiv <https://arxiv.org/abs/1910.04209>`__\n\nStable weight decay\n-------------------\n\n+-------------------------------------------------------------------------------------------------------------+\n| .. image:: https://raw.githubusercontent.com/kozistr/pytorch_optimizer/main/assets/stable_weight_decay.png  |\n+-------------------------------------------------------------------------------------------------------------+\n\n-  code : `github <https://github.com/zeke-xie/stable-weight-decay-regularization>`__\n-  paper : `arXiv <https://arxiv.org/abs/2011.11152>`__\n\nExplore-exploit learning rate schedule\n--------------------------------------\n\n+---------------------------------------------------------------------------------------------------------------------+\n| .. image:: https://raw.githubusercontent.com/kozistr/pytorch_optimizer/main/assets/explore_exploit_lr_schedule.png  |\n+---------------------------------------------------------------------------------------------------------------------+\n\n-  code : `github <https://github.com/nikhil-iyer-97/wide-minima-density-hypothesis>`__\n-  paper : `arXiv <https://arxiv.org/abs/2003.03977>`__\n\nLookahead\n---------\n\n| ``k`` steps forward, 1 step back. ``Lookahead`` consisting of keeping an exponential moving average of the weights that is\n| updated and substituted to the current weights every ``k_{lookahead}`` steps (5 by default).\n\n-  code : `github <https://github.com/alphadl/lookahead.pytorch>`__\n-  paper : `arXiv <https://arxiv.org/abs/1907.08610v2>`__\n\nChebyshev learning rate schedule\n--------------------------------\n\nAcceleration via Fractal Learning Rate Schedules\n\n-  paper : `arXiv <https://arxiv.org/abs/2103.01338v1>`__\n\n(Adaptive) Sharpness-Aware Minimization\n---------------------------------------\n\n| Sharpness-Aware Minimization (SAM) simultaneously minimizes loss value and loss sharpness.\n| In particular, it seeks parameters that lie in neighborhoods having uniformly low loss.\n\n-  SAM paper : `paper <https://arxiv.org/abs/2010.01412>`__\n-  ASAM paper : `paper <https://arxiv.org/abs/2102.11600>`__\n-  A/SAM code : `github <https://github.com/davda54/sam>`__\n\nOn the Convergence of Adam and Beyond\n-------------------------------------\n\n- paper : `paper <https://openreview.net/forum?id=ryQu7f-RZ>`__\n\nGradient Surgery for Multi-Task Learning\n----------------------------------------\n\n- paper : `paper <https://arxiv.org/abs/2001.06782>`__\n\nCitations\n---------\n\nAdamP\n\n::\n\n    @inproceedings{heo2021adamp,\n        title={AdamP: Slowing Down the Slowdown for Momentum Optimizers on Scale-invariant Weights},\n        author={Heo, Byeongho and Chun, Sanghyuk and Oh, Seong Joon and Han, Dongyoon and Yun, Sangdoo and Kim, Gyuwan and Uh, Youngjung and Ha, Jung-Woo},\n        year={2021},\n        booktitle={International Conference on Learning Representations (ICLR)},\n    }\n\nAdaptive Gradient Clipping (AGC)\n\n::\n\n    @article{brock2021high,\n      author={Andrew Brock and Soham De and Samuel L. Smith and Karen Simonyan},\n      title={High-Performance Large-Scale Image Recognition Without Normalization},\n      journal={arXiv preprint arXiv:2102.06171},\n      year={2021}\n    }\n\nChebyshev LR Schedules\n\n::\n\n    @article{agarwal2021acceleration,\n      title={Acceleration via Fractal Learning Rate Schedules},\n      author={Agarwal, Naman and Goel, Surbhi and Zhang, Cyril},\n      journal={arXiv preprint arXiv:2103.01338},\n      year={2021}\n    }\n\nGradient Centralization (GC)\n\n::\n\n    @inproceedings{yong2020gradient,\n      title={Gradient centralization: A new optimization technique for deep neural networks},\n      author={Yong, Hongwei and Huang, Jianqiang and Hua, Xiansheng and Zhang, Lei},\n      booktitle={European Conference on Computer Vision},\n      pages={635--652},\n      year={2020},\n      organization={Springer}\n    }\n\nLookahead\n\n::\n\n    @article{zhang2019lookahead,\n      title={Lookahead optimizer: k steps forward, 1 step back},\n      author={Zhang, Michael R and Lucas, James and Hinton, Geoffrey and Ba, Jimmy},\n      journal={arXiv preprint arXiv:1907.08610},\n      year={2019}\n    }\n\nRAdam\n\n::\n\n    @inproceedings{liu2019radam,\n     author = {Liu, Liyuan and Jiang, Haoming and He, Pengcheng and Chen, Weizhu and Liu, Xiaodong and Gao, Jianfeng and Han, Jiawei},\n     booktitle = {Proceedings of the Eighth International Conference on Learning Representations (ICLR 2020)},\n     month = {April},\n     title = {On the Variance of the Adaptive Learning Rate and Beyond},\n     year = {2020}\n    }\n\nNorm Loss\n\n::\n\n    @inproceedings{georgiou2021norm,\n      title={Norm Loss: An efficient yet effective regularization method for deep neural networks},\n      author={Georgiou, Theodoros and Schmitt, Sebastian and B{\\"a}ck, Thomas and Chen, Wei and Lew, Michael},\n      booktitle={2020 25th International Conference on Pattern Recognition (ICPR)},\n      pages={8812--8818},\n      year={2021},\n      organization={IEEE}\n    }\n\nPositive-Negative Momentum\n\n::\n\n    @article{xie2021positive,\n      title={Positive-Negative Momentum: Manipulating Stochastic Gradient Noise to Improve Generalization},\n      author={Xie, Zeke and Yuan, Li and Zhu, Zhanxing and Sugiyama, Masashi},\n      journal={arXiv preprint arXiv:2103.17182},\n      year={2021}\n    }\n\nExplore-Exploit learning rate schedule\n\n::\n\n    @article{iyer2020wide,\n      title={Wide-minima Density Hypothesis and the Explore-Exploit Learning Rate Schedule},\n      author={Iyer, Nikhil and Thejas, V and Kwatra, Nipun and Ramjee, Ramachandran and Sivathanu, Muthian},\n      journal={arXiv preprint arXiv:2003.03977},\n      year={2020}\n    }\n\nLinear learning-rate warm-up\n\n::\n\n    @article{ma2019adequacy,\n      title={On the adequacy of untuned warmup for adaptive optimization},\n      author={Ma, Jerry and Yarats, Denis},\n      journal={arXiv preprint arXiv:1910.04209},\n      volume={7},\n      year={2019}\n    }\n\nStable weight decay\n\n::\n\n    @article{xie2020stable,\n      title={Stable weight decay regularization},\n      author={Xie, Zeke and Sato, Issei and Sugiyama, Masashi},\n      journal={arXiv preprint arXiv:2011.11152},\n      year={2020}\n    }\n\nSoftplus transformation\n\n::\n\n    @article{tong2019calibrating,\n      title={Calibrating the adaptive learning rate to improve convergence of adam},\n      author={Tong, Qianqian and Liang, Guannan and Bi, Jinbo},\n      journal={arXiv preprint arXiv:1908.00700},\n      year={2019}\n    }\n\nMADGRAD\n\n::\n\n    @article{defazio2021adaptivity,\n      title={Adaptivity without compromise: a momentumized, adaptive, dual averaged gradient method for stochastic optimization},\n      author={Defazio, Aaron and Jelassi, Samy},\n      journal={arXiv preprint arXiv:2101.11075},\n      year={2021}\n    }\n\nAdaHessian\n\n::\n\n    @article{yao2020adahessian,\n      title={ADAHESSIAN: An adaptive second order optimizer for machine learning},\n      author={Yao, Zhewei and Gholami, Amir and Shen, Sheng and Mustafa, Mustafa and Keutzer, Kurt and Mahoney, Michael W},\n      journal={arXiv preprint arXiv:2006.00719},\n      year={2020}\n    }\n\nAdaBound\n\n::\n\n    @inproceedings{Luo2019AdaBound,\n      author = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},\n      title = {Adaptive Gradient Methods with Dynamic Bound of Learning Rate},\n      booktitle = {Proceedings of the 7th International Conference on Learning Representations},\n      month = {May},\n      year = {2019},\n      address = {New Orleans, Louisiana}\n    }\n\nAdaBelief\n\n::\n\n    @article{zhuang2020adabelief,\n      title={Adabelief optimizer: Adapting stepsizes by the belief in observed gradients},\n      author={Zhuang, Juntang and Tang, Tommy and Ding, Yifan and Tatikonda, Sekhar and Dvornek, Nicha and Papademetris, Xenophon and Duncan, James S},\n      journal={arXiv preprint arXiv:2010.07468},\n      year={2020}\n    }\n\nSharpness-Aware Minimization\n\n::\n\n    @article{foret2020sharpness,\n      title={Sharpness-aware minimization for efficiently improving generalization},\n      author={Foret, Pierre and Kleiner, Ariel and Mobahi, Hossein and Neyshabur, Behnam},\n      journal={arXiv preprint arXiv:2010.01412},\n      year={2020}\n    }\n\nAdaptive Sharpness-Aware Minimization\n\n::\n\n    @article{kwon2021asam,\n      title={ASAM: Adaptive Sharpness-Aware Minimization for Scale-Invariant Learning of Deep Neural Networks},\n      author={Kwon, Jungmin and Kim, Jeongseop and Park, Hyunseo and Choi, In Kwon},\n      journal={arXiv preprint arXiv:2102.11600},\n      year={2021}\n    }\n\ndiffGrad\n\n::\n\n    @article{dubey2019diffgrad,\n      title={diffgrad: An optimization method for convolutional neural networks},\n      author={Dubey, Shiv Ram and Chakraborty, Soumendu and Roy, Swalpa Kumar and Mukherjee, Snehasis and Singh, Satish Kumar and Chaudhuri, Bidyut Baran},\n      journal={IEEE transactions on neural networks and learning systems},\n      volume={31},\n      number={11},\n      pages={4500--4511},\n      year={2019},\n      publisher={IEEE}\n    }\n\nOn the Convergence of Adam and Beyond\n\n::\n\n    @article{reddi2019convergence,\n      title={On the convergence of adam and beyond},\n      author={Reddi, Sashank J and Kale, Satyen and Kumar, Sanjiv},\n      journal={arXiv preprint arXiv:1904.09237},\n      year={2019}\n    }\n\nGradient Surgery for Multi-Task Learning\n\n::\n\n    @article{yu2020gradient,\n      title={Gradient surgery for multi-task learning},\n      author={Yu, Tianhe and Kumar, Saurabh and Gupta, Abhishek and Levine, Sergey and Hausman, Karol and Finn, Chelsea},\n      journal={arXiv preprint arXiv:2001.06782},\n      year={2020}\n    }\n\nAdamD: Improved bias-correction in Adam\n\n::\n\n    @article{john2021adamd,\n      title={AdamD: Improved bias-correction in Adam},\n      author={John, John St},\n      journal={arXiv preprint arXiv:2110.10828},\n      year={2021}\n    }\n\nShampoo: Preconditioned Stochastic Tensor Optimization\n\n::\n\n    @inproceedings{gupta2018shampoo,\n      title={Shampoo: Preconditioned stochastic tensor optimization},\n      author={Gupta, Vineet and Koren, Tomer and Singer, Yoram},\n      booktitle={International Conference on Machine Learning},\n      pages={1842--1850},\n      year={2018},\n      organization={PMLR}\n    }\n\nNero: Learning by Turning: Neural Architecture Aware Optimisation\n\n::\n\n    @misc{nero2021,\n      title={Learning by Turning: Neural Architecture Aware Optimisation},\n      author={Yang Liu and Jeremy Bernstein and Markus Meister and Yisong Yue},\n      year={2021},\n      eprint={arXiv:2102.07227}\n    }\n\nAuthor\n------\n\nHyeongchan Kim / `@kozistr <http://kozistr.tech/about>`__\n\n.. |workflow| image:: https://github.com/kozistr/pytorch_optimizer/actions/workflows/ci.yml/badge.svg?branch=main\n.. |Documentation Status| image:: https://readthedocs.org/projects/pytorch-optimizers/badge/?version=latest\n   :target: https://pytorch-optimizers.readthedocs.io/en/latest/?badge=latest\n.. |PyPI version| image:: https://badge.fury.io/py/pytorch-optimizer.svg\n   :target: https://badge.fury.io/py/pytorch-optimizer\n.. |PyPi download| image:: https://img.shields.io/pypi/dm/pytorch-optimizer?style=plastic\n.. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg\n.. |codecov| image:: https://codecov.io/gh/kozistr/pytorch_optimizer/branch/main/graph/badge.svg?token=L4K00EA0VD\n   :target: https://codecov.io/gh/kozistr/pytorch_optimizer\n',
    'author': 'kozistr',
    'author_email': 'kozistr@gmail.com',
    'maintainer': 'kozistr',
    'maintainer_email': 'kozistr@gmail.com',
    'url': 'https://github.com/kozistr/pytorch_optimizer',
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'python_requires': '>=3.7,<4.0',
}


setup(**setup_kwargs)
