(defun address-of (array &rest indices)
  (make-array 1 :element-type (array-element-type array)
                :displaced-to array
                :displaced-index-offset (if indices
                                            (reduce (function *) indices)
                                            0)))

(defun deref (address) (aref address 0))
(defun (setf deref) (new-value address) (setf (aref address 0) new-value))

(let* ((array   (vector 1 2 3 4))
       (pointer (address-of array 2)))
  (incf (deref pointer) 100)
  array)
--> #(1 2 103 4)