diff --git a/openbox/grab.c b/openbox/grab.c
index 3e1066ba..b6a7a939 100644
--- a/openbox/grab.c
+++ b/openbox/grab.c
@@ -43,7 +43,8 @@ static Time  grab_time = CurrentTime;
 static Time ungrab_time()
 {
     Time t = event_curtime;
-    if (!(t == CurrentTime || event_time_after(t, grab_time)))
+    if (grab_time == CurrentTime ||
+        !(t == CurrentTime || event_time_after(t, grab_time)))
         /* When the time moves backward on the server, then we can't use
            the grab time because that will be in the future. So instead we
            have to use CurrentTime.