large-csv-to-kafka.clj

#!/usr/bin/env bb
;; ^^ this tells our shell to use Babashka to run this script

;; read the file path of our CSV and the key field from the command line args
(def csv-file-path (first *command-line-args*))
(def key-field (second *command-line-args*))

(with-open [reader (io/reader csv-file-path)]

;; read the CSV line-by-line into a data structure
(def csv-data
    (csv/read-csv reader))

(def headers (first csv-data))
(def body (rest csv-data))

;; For each line in the body, create a map with the headers as the keys
(def values
  (->> body
       (map (partial zipmap headers))
       ;; if you need to do any additional processing on each line, do it here
       ))

(def output-lines
  (->> values
       (map #(str (get % key-field) "::" (json/generate-string %)))))

(doseq [output output-lines]
    (println output))
)