support for modal children, both in the focus code and in the raise/lower code

src/client.cc

@@ -53,6 +53,8 @@ Client::Client(int screen, Window window)
   _positioned = false;
   // nothing is disabled unless specified
   _disabled_decorations = 0;
+  // no modal children until they set themselves
+  _modal_child = 0;
   
   getArea();
   getDesktop();
@@ -105,6 +107,8 @@ Client::~Client()
   }
   
   // clean up parents reference to this
+  if (_modal)
+    setModal(false);
   if (_transient_for)
     _transient_for->_transients.remove(this); // remove from old parent
   
@@ -674,11 +678,18 @@ void Client::updateTransientFor()
 
   // if anything has changed...
   if (c != _transient_for) {
+    bool m = _modal;
+    if (_modal)
+      setModal(false);
+    
     if (_transient_for)
       _transient_for->_transients.remove(this); // remove from old parent
     _transient_for = c;
     if (_transient_for)
       _transient_for->_transients.push_back(this); // add to new parent
+
+    if (m)
+      setModal(true);
   }
 }
 
@@ -791,12 +802,59 @@ void Client::setDesktop(long target)
 }
 
 
+Client *Client::findModalChild(Client *skip) const
+{
+  Client *ret = 0;
+  
+  // find a modal child recursively and try focus it
+  List::const_iterator it, end = _transients.end();
+  for (it = _transients.begin(); it != end; ++it)
+    if ((*it)->_modal && *it != skip)
+      return *it; // got one
+  // none of our direct children are modal, let them try check
+  for (it = _transients.begin(); it != end; ++it)
+    if ((ret = (*it)->findModalChild()))
+      return ret; // got one
+  return ret;
+}
+
+
+void Client::setModal(bool modal)
+{
+  if (modal) {
+    Client *c = this;
+    while (c->_transient_for) {
+      c = c->_transient_for;
+      if (c->_modal_child) break; // already has a modal child
+      c->_modal_child = this;
+    }
+  } else {
+    // try find a replacement modal dialog
+    Client *replacement = 0;
+    
+    Client *c = this;
+    while (c->_transient_for) // go up the tree
+      c = c->_transient_for;
+    replacement = c->findModalChild(this); // find a modal child, skipping this
+
+    c = this;
+    while (c->_transient_for) {
+      c = c->_transient_for;
+      if (c->_modal_child != this) break; // has a different modal child
+      c->_modal_child = replacement;
+    }
+  }
+  _modal = modal;
+}
+
+
 void Client::setState(StateAction action, long data1, long data2)
 {
   bool shadestate = _shaded;
   bool fsstate = _fullscreen;
   bool maxh = _max_horz;
   bool maxv = _max_vert;
+  bool modal = _modal;
 
   if (!(action == State_Add || action == State_Remove ||
         action == State_Toggle))
@@ -832,8 +890,7 @@ void Client::setState(StateAction action, long data1, long data2)
     if (action == State_Add) {
       if (state == otk::Property::atoms.net_wm_state_modal) {
         if (_modal) continue;
-        _modal = true;
+        modal = true;
-        // XXX: give it focus if another window has focus that shouldnt now
       } else if (state == otk::Property::atoms.net_wm_state_maximized_vert) {
         maxv = true;
       } else if (state == otk::Property::atoms.net_wm_state_maximized_horz) {
@@ -858,7 +915,7 @@ void Client::setState(StateAction action, long data1, long data2)
     } else { // action == State_Remove
       if (state == otk::Property::atoms.net_wm_state_modal) {
         if (!_modal) continue;
-        _modal = false;
+        modal = false;
       } else if (state == otk::Property::atoms.net_wm_state_maximized_vert) {
         maxv = false;
       } else if (state == otk::Property::atoms.net_wm_state_maximized_horz) {
@@ -895,6 +952,8 @@ void Client::setState(StateAction action, long data1, long data2)
         maximize(maxv, 2, true);
     }
   }
+  if (modal != _modal)
+    setModal(modal);
   // change fullscreen state before shading, as it will affect if the window
   // can shade or not
   if (fsstate != _fullscreen)
@@ -1303,6 +1362,11 @@ void Client::applyStartupState()
 {
   // these are in a carefully crafted order..
 
+  if (_modal) {
+    _modal = false;
+    setModal(true);
+  }
+  
   if (_iconic) {
     _iconic = false;
     setDesktop(ICONIC_DESKTOP);
@@ -1548,33 +1612,18 @@ void Client::disableDecorations(DecorationFlags flags)
 }
 
 
-bool Client::focusModalChild()
-{
-  // find a modal child recursively and try focus it
-  List::iterator it, end = _transients.end();
-  for (it = _transients.begin(); it != end; ++it)
-    if ((*it)->focusModalChild())
-      return true; // got one
-  // none of our grand-children are modal, try our direct children
-  for (it = _transients.begin(); it != end; ++it)
-    if ((*it)->modal() && (*it)->focus())
-      return true; // got one
-  return false;
-}
-
-
 bool Client::focus()
 {
+  // if we have a modal child, then focus it, not us
+  if (_modal_child)
+    return _modal_child->focus();
+
   // won't try focus if the client doesn't want it, or if the window isn't
   // visible on the screen
   if (!(frame->isVisible() && (_can_focus || _focus_notify))) return false;
 
   if (_focused) return true;
 
-  if (!_modal)
-    if (focusModalChild())
-      return true;
-
   // do a check to see if the window has already been unmapped or destroyed
   // do this intelligently while watching out for unmaps we've generated
   // (ignore_unmaps > 0)

src/client.hh

@@ -295,6 +295,9 @@ private:
   //! The window uses shape extension to be non-rectangular?
   bool _shaped;
 
+  //! If the window has a modal child window, then this will point to it
+  Client *_modal_child;
+
   //! The window is modal, so it must be processed before any windows it is
   //! related to can be focused
   bool _modal;
@@ -381,6 +384,11 @@ private:
     by sending it to any other valid desktop.
   */
   void setDesktop(long desktop);
+  //! Set whether the window is modal or not
+  /*!
+    This adjusts references in parents etc to match.
+  */
+  void setModal(bool modal);
   
   //! Calculates the stacking layer for the client window
   void calcLayer();
@@ -472,11 +480,8 @@ private:
   void internal_resize(Corner anchor, int w, int h, bool user = true,
                        int x = INT_MIN, int y = INT_MIN);
 
-  //! Attempts to focus a modal child of this window, recursively.
+  //! Attempts to find and return a modal child of this window, recursively.
-  /*!
+  Client *findModalChild(Client *skip = 0) const;
-    @return true if a modal child has been found and focused; otherwise, false.
-  */
-  bool focusModalChild();
 
 public:
 #ifndef SWIG
@@ -563,6 +568,9 @@ BB    @param window The window id that the Client class should handle
   //! Return the client this window is transient for
   inline Client *transientFor() const { return _transient_for; }
 
+  //! Returns the window which is a modal child of this window
+  inline Client *modalChild() const { return _modal_child; }
+  
   //! Returns if the window is modal
   /*!
     If the window is modal, then no other windows that it is related to can get

src/screen.cc

@@ -637,14 +637,28 @@ void Screen::lowerWindow(Client *client)
   Client::List::iterator it = --_stacking.end();
   const Client::List::iterator end = _stacking.begin();
 
-  for (; it != end && (*it)->layer() < client->layer(); --it);
+  if (client->modal() && client->transientFor()) {
-  if (*it == client) return;          // already the bottom, return
+    // don't let a modal window lower below its transient_for
+    it = std::find(_stacking.begin(), _stacking.end(), client->transientFor());
+    assert(it != _stacking.end());
 
-  wins[0] = (*it)->frame->window();
+    wins[0] = (it == _stacking.begin() ? _focuswindow :
-  wins[1] = client->frame->window();
+               ((*(--Client::List::const_iterator(it)))->frame->window()));
+    wins[1] = client->frame->window();
+    if (wins[0] == wins[1]) return; // already right above the window
 
-  _stacking.remove(client);
+    _stacking.remove(client);
-  _stacking.insert(++it, client);
+    _stacking.insert(it, client);
+  } else {
+    for (; it != end && (*it)->layer() < client->layer(); --it);
+    if (*it == client) return;          // already the bottom, return
+
+    wins[0] = (*it)->frame->window();
+    wins[1] = client->frame->window();
+
+    _stacking.remove(client);
+    _stacking.insert(++it, client);
+  }
 
   XRestackWindows(**otk::display, wins, 2);
   changeStackingList();
@@ -676,7 +690,12 @@ void Screen::raiseWindow(Client *client)
   _stacking.insert(it, client);
 
   XRestackWindows(**otk::display, wins, 2);
-  changeStackingList();
+
+  // if the window has a modal child, then raise it after us to put it on top
+  if (client->modalChild())
+    raiseWindow(client->modalChild());
+  else
+    changeStackingList(); // no need to do this twice!
 }
 
 void Screen::changeDesktop(long desktop)