(defn batch-insert-links [lls]
  (let [all (filter identity (map get-link-map lls))
        batch-size 50]
    (loop [c all]
      (let [chnk (take batch-size c)]
        (when (seq chnk)
          (try
            (j/with-db-connection [conn (mysql/get-conn)]
              (apply j/insert! conn :Links chnk))
            (catch Exception e
              (if (re-find (re-pattern "[a-zA-Z0-9]{40}") (.getMessage e))
                (future (let [clean-stuff
                              (remove #(= (re-find (re-pattern "[a-zA-Z0-9]{40}")
                                                   (.getMessage e)) (:urlhash %)) chnk)]
                          (swap! dups-vec #(cons clean-stuff %))))
                (track-error :ghost/batch-insert-links e chnk))))
          (recur (drop batch-size c))))))
  (println  "batch done"))