CS6068: Experiments with Clojure Parallel Code on oakley.o

Experiments with Clojure Parallel Code Running on oakley.osc.edu

#Connect to OSC
$ ssh oakley.osc.edu -l ucn1386

#Download and install lein
-bash-4.1$ wget https://raw.github.com/technomancy/leiningen/stable/bin/lein
-bash-4.1$ mkdir ~/bin
-bash-4.1$ cp ./lein ~/bin
-bash-4.1$ chmod +x ~/bin/lein


#Request multi-core node
-bash-4.1$ qsub -I -l nodes=1:ppn=12 -l walltime=00:20:00
qsub: waiting for job 4496542.oak-batch.osc.edu to startqsub: job 4496542.oak-batch.osc.edu ready

#Create a Clojure Project called wonderland using lein

-bash-4.1$ ~/bin/lein new wonderland
-bash-4.1$ cd wonderland/
-bash-4.1$ ~/bin/lein repl


nREPL server started on port 58060 on host 127.0.0.1 - nrepl://127.0.0.1:58060
user=> (defn long-running-job2 [n]
          (dorun (for [x (range 1000) y (range 1000)] (* x y))))

#'user/long-running-job2
user=> (time (doall (map long-running-job2 (range 20))))

"Elapsed time: 2939.689332 msecs"
user=> (time (doall (pmap long-running-job2 (range 20))))

"Elapsed time: 769.314443 msecs"

.............................4x speedup with pmap .......................

user=> (def orc-names (random-string-list 40 700000))

#'user/orc-names

user=> (time (dorun (map clojure.string/lower-case orc-names)))

"Elapsed time: 501.401414 msecs"

nil

user=> (time (dorun (pmap clojure.string/lower-case orc-names)))

"Elapsed time: 80.57307 msecs"

nil

user=> (def orc-name-abbrevs (random-string-list 1200000 300))
#'user/orc-name-abbrevs

user=> (time (dorun (map clojure.string/lower-case orc-name-abbrevs)))
"Elapsed time: 5349.481361 msecs"
nil
user=> (time (dorun (pmap clojure.string/lower-case orc-name-abbrevs)))
"Elapsed time: 2831.096441 msecs"
nil
user=> (time (dorun (apply concat
              (pmap (fn [name] (doall (map clojure.string/lower-case name)))
                              (partition-all 100000 orc-name-abbrevs)))))
"Elapsed time: 1064.08318 msecs"
nil

# Another Experiment

user=> (def orc-names (random-string-list 100000 2))
#'user/orc-names
user=> (time (reduce #(assoc %1 %2 (inc (get %1 %2 0))) {} orc-names))
"Elapsed time: 181.191269 msecs"
user=> (defn mysort [x] (into (sorted-map-by (fn [key1 key2](compare [(get x key2) key2] [(get x key1) key1]))) x))
#'user/mysort
user=> (mysort (r/reduce #(assoc %1 %2 (inc (get %1 %2 0))) {} orc-names))
{"IX" 192, "BA" 192, "XH" 183, "PH" 180, "NB" 180, "SE" 179, "KN" 178, "BL" 178, "ZJ" 177, "WJ" 175, "CP" 174, "PG" 173, "LD" 173, "EI" 173, "BD" 173, "AR" 173, "SW" 172, "JF" 172, "JE" 172, "GX" 172,
...}





-bash-4.1$ cat > mypmap.clj
;; MY-PMAP
(defn long-running-job [n]
  (Thread/sleep 3000)                   ; wait for 3 seconds
  (+ n 10))

(time (doall (map long-running-job (range 20))))
;; Elapsed time: 60053.862705 msecs
(10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29)

(time (doall (pmap long-running-job (range 20))))
;;Elapsed time: 3009.768988 msecs
(10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29)

;; This is a false test for speedup, since sleep function apparently allows
;; and arbitrary number of threads to rest simultaneously.
;; So let us define a collection active jobs using dorun.

;; dorun, doall, and doseq are all used for forcing lazy sequences, presumably to get side effects.

;; dorun - don't hold whole seq in memory while forcing, return nil
;; doall - hold whole seq in memory while forcing (i.e., all of it) and return the seq
;; doseq - same as dorun, but gives you chance to do something with each element as it's forced; returns nil


(defn long-running-job2 [n]
  (dorun  (for [x (range 1000) y (range 1000)] (* x y))))

(time (doall (map long-running-job2 (range 20))))
;; Elapsed time: 6489.148202 msecs
;;(nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil)

(time (doall (pmap long-running-job2 (range 20))))
;;Elapsed time: 3378.788034 msecs
;;(nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil)

;; The experiment above shows pmap gives approximate 2x speedup.

;;String processing
(def alphabet-length 26)

;; vector of chars, A-Z
(def letters (mapv (comp str char (partial + 65)) (range alphabet-length)))

(defn random-string
;; returns a random string of specified length
  [length]
  (apply str (take length (repeatedly #(rand-nth letters)))))

(defn random-string-list
  [list-length string-length]
  (doall (take list-length (repeatedly (partial random-string string-length)))))

(def orc-names (random-string-list 40 70000))


;; Use `dorun` to realize the lazy seq returned by map without
;; printing the results in the REPL
(time (dorun (map clojure.string/lower-case orc-names)))

(time (dorun (pmap clojure.string/lower-case orc-names)))


;;Partitioning for grain size
(def numbers [1 2 3 4 5 6 7 8 9 10])
(partition-all 3 numbers)

(pmap inc numbers)  ; grain size 1

(pmap (fn [number-group] (doall (map inc number-group)))
      (partition-all 3 numbers))


(apply concat
       (pmap (fn [number-group] (doall (map inc number-group)))
             (partition-all 3 numbers)))

(def orc-name-abbrevs (random-string-list 200000 300))

(time (dorun (map clojure.string/lower-case orc-name-abbrevs)))

(time
 (dorun
  (apply concat
         (pmap (fn [name] (doall (map clojure.string/lower-case name)))
               (partition-all 100000 orc-name-abbrevs)))))