AISTATS 2023poster1 citations
Coarse-Grained Smoothness for Reinforcement Learning in Metric Spaces
Omer Gottesman, Kavosh Asadi, Cameron S. Allen, Samuel Lobel, George Konidaris, Michael Littman
Abstract
Principled decision-making in continuous state–action spaces is impossible without some assumptions. A common approach is to assume Lipschitz continuity of the Q-function. We show that, unfortunately, this property fails to hold in many typical domains. We propose a new coarse-grained smoothness definition that generalizes the notion of Lipschitz continuity, is more widely applicable, and allows us to compute significantly tighter bounds on Q-functions, leading to improved learning. We provide a theoretical analysis of our new smoothness definition, and discuss its implications and impact on control and exploration in continuous domains.
BibTeX
@InProceedings{pmlr-v206-gottesman23a,
title = {Coarse-Grained Smoothness for Reinforcement Learning in Metric Spaces},
author = {Gottesman, Omer and Asadi, Kavosh and Allen, Cameron S. and Lobel, Samuel and Konidaris, George and Littman, Michael},
booktitle = {Proceedings of The 26th International Conference on Artificial Intelligence and Statistics},
pages = {1390--1410},
year = {2023},
editor = {Ruiz, Francisco and Dy, Jennifer and van de Meent, Jan-Willem},
volume = {206},
series = {Proceedings of Machine Learning Research},
month = {25--27 Apr},
publisher = {PMLR},
pdf = {https://proceedings.mlr.press/v206/gottesman23a/gottesman23a.pdf},
url = {https://proceedings.mlr.press/v206/gottesman23a.html},
abstract = {Principled decision-making in continuous state–action spaces is impossible without some assumptions. A common approach is to assume Lipschitz continuity of the Q-function. We show that, unfortunately, this property fails to hold in many typical domains. We propose a new coarse-grained smoothness definition that generalizes the notion of Lipschitz continuity, is more widely applicable, and allows us to compute significantly tighter bounds on Q-functions, leading to improved learning. We provide a theoretical analysis of our new smoothness definition, and discuss its implications and impact on control and exploration in continuous domains.}
}