iostat_parser.pig

/*
iostat -dxk 1 | awk '{OFS="\t"}{print $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13}' > 1_second_smapler_disk_slave04.data

(Device:,rrqm/s,wrqm/s,r/s,w/s,rkB/s,wkB/s,avgrq-sz,avgqu-sz,await,r_await,w_await,svctm)
(sda,0.03,0.12,0.03,1.12,0.65,9.92,18.27,0.00,0.62,5.90,0.47,0.23)

 AVG(filtered.avgqusz) as avg_avgqusz, MAX(filtered.avgqusz) as max_avgqusz,
*/

raw = LOAD '/project/jsn/diskio/dev_slave01_after_compress/*' using PigStorage('\t') as (device:chararray, rrqms:float, wrqms:float, rs:float, ws:float, rkBs:float, wkBs:float, avgrqsz:float, avgqusz:float, await:float, r_await:float, w_await:float, svctm:float);
filtered = FILTER raw BY svctm is not null;
device_groups = GROUP filtered BY device;
device_stats = FOREACH device_groups GENERATE group, AVG(filtered.await) as avg_await, MAX(filtered.await) as max_await, AVG(filtered.svctm) as avg_svctm, MAX(filtered.svctm) as max_svctm;
dump device_stats
/*
DEVICE, await avg, await max, scvtm avg, scvtm max
dev slave 4 before compress
(sda,4.719613106522177,942.69,0.516785713954873,34.25)
(sdb,133.26059492145265,1448.58,10.88648805589903,22.22)
(sdc,19.647827405127742,119.32,7.606994046341805,28.13)
(sdd,98.69758911076046,1257.12,10.319017841702415,16.18)
(sde,47.0387501979158,939.92,8.249940480859507,13.75)

dev slave 1 after compress
(sda,0.9336349939306577,82.0,0.6378883862050411,82.0)
(sdb,41.89456335679594,836.89,9.964367494346147,65.1)
(sdc,37.484102510938456,267.62,10.321176473910992,35.58)
(sdd,29.441749674162715,674.2,9.468567108316838,43.94)
(sde,26.91143283153551,473.27,9.00064857944644,32.68)


prod slave05 before compression
(sda,0.6517554233199687,40.0,0.27250493138374604,40.0)
(sdb,9.116499011943032,79.5,3.8455522651627985,69.0)
(sdc,21.455512780732892,933.41,4.160848125896217,86.71)
(sdd,8.752001977589943,60.31,3.3266962531373583,24.0)
(sde,9.198864753417825,95.0,3.737660413300885,83.0)
(sdf,5.9169299054122275,89.0,3.699782823200493,89.0)
(sdg,8.920226833380802,183.8,3.452790927259468,58.0)
(sdh,8.693484703650714,114.0,3.321490623108373,68.0)
(sdi,7.661085880726487,60.0,2.8072260619515346,41.0)
(sdj,8.30831194260072,38.12,3.343425469309356,48.0)
(sdk,7.787512340173787,122.5,2.7687068117629856,97.0)
(sdl,7.9183909102524135,94.15,2.945004934810933,22.0)
(sdm,10.629634749677551,186.93,3.0339684098341153,21.0)





*/