AISTATS 2018poster0 citations
Natural Gradients in Practice: Non-Conjugate Variational Inference in Gaussian Process Models
Hugh Salimbeni, Stefanos Eleftheriadis, James Hensman
Abstract
The natural gradient method has been used effectively in conjugate Gaussian process models, but the non-conjugate case has been largely unexplored. We examine how natural gradients can be used in non-conjugate stochastic settings, together with hyperparameter learning. We conclude that the natural gradient can significantly improve performance in terms of wall-clock time. For ill-conditioned posteriors the benefit of the natural gradient method is especially pronounced, and we demonstrate a practical setting where ordinary gradients are unusable. We show how natural gradients can be computed efficiently and automatically in any parameterization, using automatic differentiation.
BibTeX
@InProceedings{pmlr-v84-salimbeni18a,
title = {Natural Gradients in Practice: Non-Conjugate Variational Inference in Gaussian Process Models},
author = {Salimbeni, Hugh and Eleftheriadis, Stefanos and Hensman, James},
booktitle = {Proceedings of the Twenty-First International Conference on Artificial Intelligence and Statistics},
pages = {689--697},
year = {2018},
editor = {Storkey, Amos and Perez-Cruz, Fernando},
volume = {84},
series = {Proceedings of Machine Learning Research},
month = {09--11 Apr},
publisher = {PMLR},
pdf = {http://proceedings.mlr.press/v84/salimbeni18a/salimbeni18a.pdf},
url = {https://proceedings.mlr.press/v84/salimbeni18a.html},
abstract = {The natural gradient method has been used effectively in conjugate Gaussian process models, but the non-conjugate case has been largely unexplored. We examine how natural gradients can be used in non-conjugate stochastic settings, together with hyperparameter learning. We conclude that the natural gradient can significantly improve performance in terms of wall-clock time. For ill-conditioned posteriors the benefit of the natural gradient method is especially pronounced, and we demonstrate a practical setting where ordinary gradients are unusable. We show how natural gradients can be computed efficiently and automatically in any parameterization, using automatic differentiation. }
}