ICML 2019oral34 citations
On Dropout and Nuclear Norm Regularization
Abstract
We give a formal and complete characterization of the explicit regularizer induced by dropout in deep linear networks with squared loss. We show that (a) the explicit regularizer is composed of an $\ell_2$-path regularizer and other terms that are also re-scaling invariant, (b) the convex envelope of the induced regularizer is the squared nuclear norm of the network map, and (c) for a sufficiently large dropout rate, we characterize the global optima of the dropout objective. We validate our theoretical findings with empirical results.
BibTeX
@InProceedings{pmlr-v97-mianjy19a,
title = {On Dropout and Nuclear Norm Regularization},
author = {Mianjy, Poorya and Arora, Raman},
booktitle = {Proceedings of the 36th International Conference on Machine Learning},
pages = {4575--4584},
year = {2019},
editor = {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
volume = {97},
series = {Proceedings of Machine Learning Research},
month = {09--15 Jun},
publisher = {PMLR},
pdf = {http://proceedings.mlr.press/v97/mianjy19a/mianjy19a.pdf},
url = {https://proceedings.mlr.press/v97/mianjy19a.html},
abstract = {We give a formal and complete characterization of the explicit regularizer induced by dropout in deep linear networks with squared loss. We show that (a) the explicit regularizer is composed of an $\ell_2$-path regularizer and other terms that are also re-scaling invariant, (b) the convex envelope of the induced regularizer is the squared nuclear norm of the network map, and (c) for a sufficiently large dropout rate, we characterize the global optima of the dropout objective. We validate our theoretical findings with empirical results.}
}