this refinement gets rid of the rw lock in the I/O path and provides a bit
more fairness to open/upgrade/downgrade/close and uses dec_and_lock to
better prevent spurious wakeup due to a race that is not habdled above

increment io_counter
if fd_counter != 0
    lock mutex
    if fd_counter != 0
        // fd work is waiting, block until it's done
        if decrement io_counter == 0
            signal condition var
        while fd_counter != 0
            wait on condition var
        // we are done waiting for fd work, so resume I/O
        increment io_counter
    unlock mutex
// now we know no fd work is waiting or in progress and can't start
initiate I/O

on I/O complete (async or immediate):
-------------------------------------

if decrement_and_lock io_counter
    if fd_counter != 0
        signal condition var
    unlock mutex

open/upgrade/downgrade/close:
-----------------------------

increment fd_counter
lock mutex
while io_counter != 0
    // now fd_counter is non-zero and io_counter is zero, any I/O that
    // tries to start will block until fd_counter goes to zero
    wait on cond var

// io_counter was zero, and because we checked while holding the mutex, but made
// fd_counter non-zero before taking the mutex any I/O threads that try to start
// after we took the mutex MUST be blocked, proceed
unlock mutex

// serialize any open/upgrade/downgrade/close
take write lock
re-open fd in new mode or close it
release write lock

// now ready to allow I/O to resume
if decrement_and_lock fd_counter
    if fd_counter == 0
        signal cond var
    unlock mutex