ICML 2017poster350 citations
Stochastic Modified Equations and Adaptive Stochastic Gradient Algorithms
Qianxiao Li, Cheng Tai, Weinan E
Abstract
We develop the method of stochastic modified equations (SME), in which stochastic gradient algorithms are approximated in the weak sense by continuous-time stochastic differential equations. We exploit the continuous formulation together with optimal control theory to derive novel adaptive hyper-parameter adjustment policies. Our algorithms have competitive performance with the added benefit of being robust to varying models and datasets. This provides a general methodology for the analysis and design of stochastic gradient algorithms.
BibTeX
@InProceedings{pmlr-v70-li17f,
title = {Stochastic Modified Equations and Adaptive Stochastic Gradient Algorithms},
author = {Qianxiao Li and Cheng Tai and Weinan E},
booktitle = {Proceedings of the 34th International Conference on Machine Learning},
pages = {2101--2110},
year = {2017},
editor = {Precup, Doina and Teh, Yee Whye},
volume = {70},
series = {Proceedings of Machine Learning Research},
month = {06--11 Aug},
publisher = {PMLR},
pdf = {http://proceedings.mlr.press/v70/li17f/li17f.pdf},
url = {https://proceedings.mlr.press/v70/li17f.html},
abstract = {We develop the method of stochastic modified equations (SME), in which stochastic gradient algorithms are approximated in the weak sense by continuous-time stochastic differential equations. We exploit the continuous formulation together with optimal control theory to derive novel adaptive hyper-parameter adjustment policies. Our algorithms have competitive performance with the added benefit of being robust to varying models and datasets. This provides a general methodology for the analysis and design of stochastic gradient algorithms.}
}