Mar 20 20:23:39 test1 kernel: mpt2sas_cm0: log_info(0x31080000): originator(PL), code(0x08), sub_code(0x0000)
Mar 20 20:23:39 test1 kernel: sd 0:0:5:0: [sdf] FAILED Result: hostbyte=DID_SOFT_ERROR driverbyte=DRIVER_OK
Mar 20 20:23:39 test1 kernel: sd 0:0:5:0: [sdf] CDB: Read(16) 88 00 00 00 00 01 3c 56 1e 72 00 00 00 69 00 00
Mar 20 20:23:39 test1 kernel: blk_update_request: I/O error, dev sdf, sector 5307244146
Mar 20 20:23:39 test1 kernel: sd 0:0:5:0: [sdf] FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Mar 20 20:23:39 test1 kernel: sd 0:0:5:0: [sdf] Sense Key : Medium Error [current] [descriptor]
Mar 20 20:23:39 test1 kernel: sd 0:0:5:0: [sdf] Add. Sense: Unrecovered read error
Mar 20 20:23:39 test1 kernel: sd 0:0:5:0: [sdf] CDB: Read(16) 88 00 00 00 00 01 3c 56 1d af 00 00 00 c3 00 00
Mar 20 20:23:39 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 5307243951


[root@test1 log]# cat /var/log/messages |grep "medium error"
Mar 20 19:16:39 test1 kernel: blk_update_request: critical medium error, dev sdc, sector 4707459812
Mar 20 20:23:39 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 5307243951
Mar 20 20:44:03 test1 kernel: blk_update_request: critical medium error, dev sdc, sector 6754670914
Mar 20 21:29:34 test1 kernel: blk_update_request: critical medium error, dev sde, sector 1499462270
Mar 21 00:16:46 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 1149755218
Mar 21 00:36:18 test1 kernel: blk_update_request: critical medium error, dev sdc, sector 3065992669
Mar 21 00:52:42 test1 kernel: blk_update_request: critical medium error, dev sdc, sector 5320325772
Mar 21 02:05:57 test1 kernel: blk_update_request: critical medium error, dev sda, sector 5105039980
Mar 21 02:19:21 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 2463285899
Mar 21 02:22:55 test1 kernel: blk_update_request: critical medium error, dev sdc, sector 7395031073
Mar 21 02:23:18 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 9488242914
Mar 21 04:07:12 test1 kernel: blk_update_request: critical medium error, dev sdb, sector 3471101487
Mar 21 04:15:54 test1 kernel: blk_update_request: critical medium error, dev sda, sector 3478310954
Mar 21 05:56:50 test1 kernel: blk_update_request: critical medium error, dev sda, sector 5201337353
Mar 21 05:58:36 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 254572506
Mar 21 06:32:37 test1 kernel: blk_update_request: critical medium error, dev sdc, sector 1888604824
Mar 21 08:26:09 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 771670692
Mar 21 09:16:07 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 3104221097
Mar 21 09:25:36 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 820117023
Mar 21 10:36:25 test1 kernel: blk_update_request: critical medium error, dev sda, sector 1163472446
Mar 21 10:59:53 test1 kernel: blk_update_request: critical medium error, dev sda, sector 1170459050
Mar 21 11:02:55 test1 kernel: blk_update_request: critical medium error, dev sdb, sector 3320463643
Mar 21 18:17:21 test1 kernel: blk_update_request: critical medium error, dev sdb, sector 7399667902
Mar 21 18:57:06 test1 kernel: blk_update_request: critical medium error, dev sdb, sector 6623511463
Mar 21 19:19:06 test1 kernel: blk_update_request: critical medium error, dev sdc, sector 8988759994
Mar 21 20:43:46 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 1186775277
Mar 21 23:10:38 test1 kernel: blk_update_request: critical medium error, dev sda, sector 8970151738
Mar 22 00:28:00 test1 kernel: blk_update_request: critical medium error, dev sde, sector 1088844219
Mar 22 03:27:35 test1 kernel: blk_update_request: critical medium error, dev sdb, sector 9217320303
Mar 22 04:54:08 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 1355466098
Mar 22 06:49:14 test1 kernel: blk_update_request: critical medium error, dev sda, sector 3312336958
Mar 22 08:09:49 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 3049018341
Mar 22 08:15:34 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 3065992666
Mar 22 08:28:38 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 3102693890
Mar 22 08:44:51 test1 kernel: blk_update_request: critical medium error, dev sda, sector 831675483
Mar 22 09:08:35 test1 kernel: blk_update_request: critical medium error, dev sdd, sector 5076166921
Mar 22 10:16:44 test1 kernel: blk_update_request: critical medium error, dev sdf, sector 7469564319


[root@test1 log]# cat /var/log/messages |grep "medium error" |cut -b 90-|sort -n|uniq -c
      1 254572506
      1 771670692
      1 820117023
      1 831675483
      1 1088844219
      1 1149755218
      1 1163472446
      1 1170459050
      1 1186775277
      1 1355466098
      1 1499462270
      1 1888604824
      1 2463285899
      1 3049018341
      1 3065992666
      1 3065992669
      1 3102693890
      1 3104221097
      1 3312336958
      1 3320463643
      1 3471101487
      1 3478310954
      1 4707459812
      1 5076166921
      1 5105039980
      1 5201337353
      1 5307243951
      1 5320325772
      1 6623511463
      1 6754670914
      1 7395031073
      1 7399667902
      1 7469564319
      1 8970151738
      1 8988759994
      1 9217320303
      1 9488242914

[root@test1 log]# for drive in sda sdb sdc sdd sde sdf;do echo $drive; cat /var/log/messages |grep "medium error" |cut -b 90-|sort -n |while read sector ; do hdparm --read-sector $sector /dev/$drive;done|grep reading;done
sda
reading sector 254572506: succeeded
reading sector 771670692: succeeded
reading sector 820117023: succeeded
reading sector 831675483: succeeded
reading sector 1088844219: succeeded
reading sector 1149755218: succeeded
reading sector 1163472446: succeeded
reading sector 1170459050: succeeded
reading sector 1186775277: succeeded
reading sector 1355466098: succeeded
reading sector 1499462270: succeeded
reading sector 1888604824: succeeded
reading sector 2463285899: succeeded
reading sector 3049018341: succeeded
reading sector 3065992666: succeeded
reading sector 3065992669: succeeded
reading sector 3102693890: succeeded
reading sector 3104221097: succeeded
reading sector 3312336958: succeeded
reading sector 3320463643: succeeded
reading sector 3471101487: succeeded
reading sector 3478310954: succeeded
reading sector 4707459812: succeeded
reading sector 5076166921: succeeded
reading sector 5105039980: succeeded
reading sector 5201337353: succeeded
reading sector 5307243951: succeeded
reading sector 5320325772: succeeded
reading sector 6623511463: succeeded
reading sector 6754670914: succeeded
reading sector 7395031073: succeeded
reading sector 7399667902: succeeded
reading sector 7469564319: succeeded
reading sector 8970151738: succeeded
reading sector 8988759994: succeeded
reading sector 9217320303: succeeded
reading sector 9488242914: succeeded
sdb
reading sector 254572506: succeeded
reading sector 771670692: succeeded
reading sector 820117023: succeeded
reading sector 831675483: succeeded
reading sector 1088844219: succeeded
reading sector 1149755218: succeeded
reading sector 1163472446: succeeded
reading sector 1170459050: succeeded
reading sector 1186775277: succeeded
reading sector 1355466098: succeeded
reading sector 1499462270: succeeded
reading sector 1888604824: succeeded
reading sector 2463285899: succeeded
reading sector 3049018341: succeeded
reading sector 3065992666: succeeded
reading sector 3065992669: succeeded
reading sector 3102693890: succeeded
reading sector 3104221097: succeeded
reading sector 3312336958: succeeded
reading sector 3320463643: succeeded
reading sector 3471101487: succeeded
reading sector 3478310954: succeeded
reading sector 4707459812: succeeded
reading sector 5076166921: succeeded
reading sector 5105039980: succeeded
reading sector 5201337353: succeeded
reading sector 5307243951: succeeded
reading sector 5320325772: succeeded
reading sector 6623511463: succeeded
reading sector 6754670914: succeeded
reading sector 7395031073: succeeded
reading sector 7399667902: succeeded
reading sector 7469564319: succeeded
reading sector 8970151738: succeeded
reading sector 8988759994: succeeded
reading sector 9217320303: succeeded
reading sector 9488242914: succeeded
sdc
reading sector 254572506: succeeded
reading sector 771670692: succeeded
reading sector 820117023: succeeded
reading sector 831675483: succeeded
reading sector 1088844219: succeeded
reading sector 1149755218: succeeded
reading sector 1163472446: succeeded
reading sector 1170459050: succeeded
reading sector 1186775277: succeeded
reading sector 1355466098: succeeded
reading sector 1499462270: succeeded
reading sector 1888604824: succeeded
reading sector 2463285899: succeeded
reading sector 3049018341: succeeded
reading sector 3065992666: succeeded
reading sector 3065992669: succeeded
reading sector 3102693890: succeeded
reading sector 3104221097: succeeded
reading sector 3312336958: succeeded
reading sector 3320463643: succeeded
reading sector 3471101487: succeeded
reading sector 3478310954: succeeded
reading sector 4707459812: succeeded
reading sector 5076166921: succeeded
reading sector 5105039980: succeeded
reading sector 5201337353: succeeded
reading sector 5307243951: succeeded
reading sector 5320325772: succeeded
reading sector 6623511463: succeeded
reading sector 6754670914: succeeded
reading sector 7395031073: succeeded
reading sector 7399667902: succeeded
reading sector 7469564319: succeeded
reading sector 8970151738: succeeded
reading sector 8988759994: succeeded
reading sector 9217320303: succeeded
reading sector 9488242914: succeeded
sdd
reading sector 254572506: succeeded
reading sector 771670692: succeeded
reading sector 820117023: succeeded
reading sector 831675483: succeeded
reading sector 1088844219: succeeded
reading sector 1149755218: succeeded
reading sector 1163472446: succeeded
reading sector 1170459050: succeeded
reading sector 1186775277: succeeded
reading sector 1355466098: succeeded
reading sector 1499462270: succeeded
reading sector 1888604824: succeeded
reading sector 2463285899: succeeded
reading sector 3049018341: succeeded
reading sector 3065992666: succeeded
reading sector 3065992669: succeeded
reading sector 3102693890: succeeded
reading sector 3104221097: succeeded
reading sector 3312336958: succeeded
reading sector 3320463643: succeeded
reading sector 3471101487: succeeded
reading sector 3478310954: succeeded
reading sector 4707459812: succeeded
reading sector 5076166921: succeeded
reading sector 5105039980: succeeded
reading sector 5201337353: succeeded
reading sector 5307243951: succeeded
reading sector 5320325772: succeeded
reading sector 6623511463: succeeded
reading sector 6754670914: succeeded
reading sector 7395031073: succeeded
reading sector 7399667902: succeeded
reading sector 7469564319: succeeded
reading sector 8970151738: succeeded
reading sector 8988759994: succeeded
reading sector 9217320303: succeeded
reading sector 9488242914: succeeded
sde
reading sector 254572506: succeeded
reading sector 771670692: succeeded
reading sector 820117023: succeeded
reading sector 831675483: succeeded
reading sector 1088844219: succeeded
reading sector 1149755218: succeeded
reading sector 1163472446: succeeded
reading sector 1170459050: succeeded
reading sector 1186775277: succeeded
reading sector 1355466098: succeeded
reading sector 1499462270: succeeded
reading sector 1888604824: succeeded
reading sector 2463285899: succeeded
reading sector 3049018341: succeeded
reading sector 3065992666: succeeded
reading sector 3065992669: succeeded
reading sector 3102693890: succeeded
reading sector 3104221097: succeeded
reading sector 3312336958: succeeded
reading sector 3320463643: succeeded
reading sector 3471101487: succeeded
reading sector 3478310954: succeeded
reading sector 4707459812: succeeded
reading sector 5076166921: succeeded
reading sector 5105039980: succeeded
reading sector 5201337353: succeeded
reading sector 5307243951: succeeded
reading sector 5320325772: succeeded
reading sector 6623511463: succeeded
reading sector 6754670914: succeeded
reading sector 7395031073: succeeded
reading sector 7399667902: succeeded
reading sector 7469564319: succeeded
reading sector 8970151738: succeeded
reading sector 8988759994: succeeded
reading sector 9217320303: succeeded
reading sector 9488242914: succeeded
sdf
reading sector 254572506: succeeded
reading sector 771670692: succeeded
reading sector 820117023: succeeded
reading sector 831675483: succeeded
reading sector 1088844219: succeeded
reading sector 1149755218: succeeded
reading sector 1163472446: succeeded
reading sector 1170459050: succeeded
reading sector 1186775277: succeeded
reading sector 1355466098: succeeded
reading sector 1499462270: succeeded
reading sector 1888604824: succeeded
reading sector 2463285899: succeeded
reading sector 3049018341: succeeded
reading sector 3065992666: succeeded
reading sector 3065992669: succeeded
reading sector 3102693890: succeeded
reading sector 3104221097: succeeded
reading sector 3312336958: succeeded
reading sector 3320463643: succeeded
reading sector 3471101487: succeeded
reading sector 3478310954: succeeded
reading sector 4707459812: succeeded
reading sector 5076166921: succeeded
reading sector 5105039980: succeeded
reading sector 5201337353: succeeded
reading sector 5307243951: succeeded
reading sector 5320325772: succeeded
reading sector 6623511463: succeeded
reading sector 6754670914: succeeded
reading sector 7395031073: succeeded
reading sector 7399667902: succeeded
reading sector 7469564319: succeeded
reading sector 8970151738: succeeded
reading sector 8988759994: succeeded
reading sector 9217320303: succeeded
reading sector 9488242914: succeeded