blktrace block device tool

Introduction
blktrace is block layer IO tracing mechanism while provides detailed information about request queue operations up to user space

Install on RedHat/CentOS/OEL
Download epel-release-6-8.noarch.rpm and install then using below command.
#yum install blktrace

Setup
#mount -t debugfs debugfs /sys/kernel/debug
Add an entry into /etc/fstab, and have it done automatically at each boot.
debug /sys/kernel/debug debugfs debugfs 0 0

blktrace – live
[root@FSS-H13-102 ~]# blktrace  -d /dev/ram0 -o – | blkparse  -i –
  1,0    4        1     0.000000000  6516  Q   R 1 + 1 [(null)]
  1,0    2        1 1266874889.709410185  6516  Q   R 0 + 1 [bksc_exec]
  1,0    2        2    12.002501106  6516  Q   R 0 + 1 [bksc_exec]
  1,0    0        1    12.002557330  6516  Q   R 1 + 1 [bksc_exec]
  1,0    1        1    13.434488103  6558  Q   R 20480 + 32 [kdownstream/1]
  1,0    1        2    13.434928252  6558  Q   R 942072 + 8 [kdownstream/1]
  1,0    1        3    13.435318671  6558  Q   R 20480 + 32 [kdownstream/1]
  1,0    1        4    13.435792065  6558  Q   R 20512 + 64 [kdownstream/1]