(defun allow-toggle-timestamp-p ()
  (cond
   ((eq last-command 'org-time-stamp) t)
   ((eq last-command 'org-toggle-timestamp-type) t)
   nil))

(define-key org-mode-map "!"
  `(menu-item "" org-toggle-timestamp-type
              :filter ,(lambda (cmd) (if (allow-toggle-timestamp-p) cmd))))