1、帶寬測試
在server端執行
[ibtests]# ib_send_bw -a -c UD -d mlx4_0 -i 1
------------------------------------------------------------------
Send BW Test
Connection type : UD
Inline data is used up to 1 bytes message
local address: LID 0x0b, QPN 0x28004c, PSN 0xfaa100
remote address: LID 0x02, QPN 0x70004b, PSN 0xc14da8
Mtu : 2048
------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec]
------------------------------------------------------------------
在client端執行
[ibtests]# ib_send_bw -a -c UD -d mlx4_0 -i 1 10.10.11.8
------------------------------------------------------------------
Send BW Test
Connection type : UD
Inline data is used up to 1 bytes message
local address: LID 0x02, QPN 0x70004b, PSN 0xc14da8
remote address: LID 0x0b, QPN 0x28004c, PSN 0xfaa100
Mtu : 2048
------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec]
2 1000 7.51 7.21
4 1000 15.29 14.19
8 1000 30.66 30.45
16 1000 60.33 59.95
32 1000 119.53 113.20
64 1000 233.75 233.16
128 1000 414.95 413.64
256 1000 794.90 698.20
512 1000 1600.46 774.67
1024 1000 2011.81 804.29
2048 1000 2923.29 2919.91
------------------------------------------------------------------
2、延時測試
在server端執行
[ibtests]# ib_send_lat -a -c UD -d mlx4_0 -i 1
------------------------------------------------------------------
Send Latency Test
Inline data is used up to 400 bytes message
Connection type : UD
local address: LID 0x0b QPN 0x2c004c PSN 0xa1be86
remote address: LID 0x02 QPN 0x74004b PSN 0x6ea837
------------------------------------------------------------------
#bytes #iterations t_min[usec] t_max[usec] t_typical[usec]
2 1000 1.41 4.45 1.43
4 1000 1.41 3.84 1.43
8 1000 1.41 2.75 1.43
16 1000 1.41 3.01 1.42
32 1000 1.49 3.92 1.50
64 1000 1.55 3.96 1.57
128 1000 1.70 2.58 1.71
256 1000 2.41 5.73 2.45
512 1000 2.82 4.07 2.90
1024 1000 3.28 4.95 3.31
2048 1000 4.11 11.74 4.14
------------------------------------------------------------------
在client端執行
[ibtests]# ib_send_lat -a -c UD -d mlx4_0 -i 2 10.10.11.8
------------------------------------------------------------------
Send Latency Test
Inline data is used up to 400 bytes message
Connection type : UD
local address: LID 0x02 QPN 0x74004b PSN 0x6ea837
remote address: LID 0x0b QPN 0x2c004c PSN 0xa1be86
------------------------------------------------------------------
#bytes #iterations t_min[usec] t_max[usec] t_typical[usec]
2 1000 1.41 9.97 1.43
4 1000 1.38 5.31 1.43
8 1000 1.41 2.78 1.43
16 1000 1.40 4.01 1.42
32 1000 1.49 3.67 1.50
64 1000 1.55 5.20 1.56
128 1000 1.69 3.13 1.71
256 1000 2.40 5.72 2.45
512 1000 2.83 4.13 2.90
1024 1000 3.28 4.95 3.31
2048 1000 4.11 11.68 4.14
------------------------------------------------------------------
2、其他測試工具
#qperf
#server端執行
[root@server ~]# qperf
#client端執行 [root@client ~]# qperf 172.26.2.41 ud_lat ud_bw rc_rdma_read_bw rc_rdma_write_bw uc_rdma_write_bw tcp_bw tcp_lat udp_bw udp_lat ud_lat: latency = 4.41 us ud_bw: send_bw = 2.63 GB/sec recv_bw = 2.63 GB/sec rc_rdma_read_bw: bw = 3.31 GB/sec rc_rdma_write_bw: bw = 3.41 GB/sec uc_rdma_write_bw: send_bw = 3.4 GB/sec recv_bw = 3.36 GB/sec tcp_bw: bw = 2.11 GB/sec tcp_lat: latency = 8.56 us udp_bw: send_bw = 2.84 GB/sec recv_bw = 699 MB/sec udp_lat: latency = 8.03 us
#iperf3
#server端執行
[root@server ~]# iperf3 -s -p 10081
#client端執行
[tpsa@client ~]$ iperf3 -c 172.26.2.41 -t 300 -p 10081
3、網絡調優
#啟用connected模式（默認是datagram模式，datagram模式下網絡延時更低，connected模式下網絡帶寬更高），接口帶寬提高一倍左右 echo connected > /sys/class/net/ib0/mode or sed -i 's/SET_IPOIB_CM=.*/SET_IPOIB_CM=yes' /etc/infiniband/openib.conf /etc/init.d/openibd restart
#系統參數調優（centos7） systemctl status tuned.service #看看是否啟用了tuned服務 tuned-adm profile network-throughput #優化網絡帶寬 tuned-adm profile network-latency #優化網絡延時 tuned-adm active #查看當前配置
# 停止irqbalance服務 # systemctl stop irqbalance && systemctl disable irqbalance #查看ib接口與哪個cpu相鄰 #numa_num=$(cat /sys/class/net/ib0/device/numa_node) #對ib網卡中斷做綁核操作 #/usr/sbin/set_irq_affinity_bynode.sh $numa_num ib0 #[root@server ~]$ rpm -qf /usr/sbin/set_irq_affinity_bynode.sh mlnx-ofa_kernel-3.3-OFED.3.3.1.0.0.1.gf583963.rhel7u2.x86_64 #驗證綁核 #查看ib0使用的中斷號 [root@server ~]# ls /sys/class/net/ib0/device/msi_irqs 100 102 104 55 57 59 61 63 65 67 69 71 75 77 79 81 83 85 87 89 91 93 95 97 99 101 103 54 56 58 60 62 64 66 68 70 74 76 78 80 82 84 86 88 90 92 94 96 98 #查看某個中斷號的smp_affinity值 [root@server ~]# cat /proc/irq/100/smp_affinity 0000,00001000 #跟默認值對比 [root@server ~]# cat /proc/irq/default_smp_affinity
#也可以通過mellanox提供的工具自動優化 # mlnx_tune -h Usage: mlnx_tune [options] Options: -h, --help show this help message and exit -d, --debug_info dump system debug information without setting a profile -r, --report Report HW/SW status and issues without setting a profile -c, --colored Switch using colored/monochromed status reports. Only applicable with --report -p PROFILE, --profile=PROFILE Set profile and run it. choose from: ['HIGH_THROUGHPUT', 'IP_FORWARDING_MULTI_STREAM_THROUGHPUT', 'IP_FORWARDING_MULTI_STREAM_PACKET_RATE', 'IP_FORWARDING_SINGLE_STREAM', 'IP_FORWARDING_SINGLE_STREAM_0_LOSS', 'IP_FORWARDING_SINGLE_STREAM_SINGLE_PORT', 'LOW_LATENCY_VMA'] -q, --verbosity print debug information to the screen [default False] -v, --version print tool version and exit [default False] -i INFO_FILE_PATH, --info_file_path=INFO_FILE_PATH info_file path. [default %s] #顯示當前配置狀態 # mlnx_tune -r #開始優化， # mlnx_tune -p HIGH_THROUGHPUT [root@server ~]# rpm -qf `which mlnx_tune` mlnx-ofa_kernel-3.3-OFED.3.3.1.0.0.1.gf583963.rhel7u2.x86_64
3、查看接口信息
[root@gz-cs-gpu-3-8 eden]# ibstat
CA 'mlx4_0'
CA type: MT26428
Number of ports: 1
Firmware version: 2.9.1000
Hardware version: b0
Node GUID: 0x0002c9030059ddda
System image GUID: 0x0002c9030059dddd
Port 1:
State: Active
Physical state: LinkUp
Rate: 40
Base lid: 58
LMC: 0
SM lid: 1
Capability mask: 0x02510868
Port GUID: 0x0002c9030059dddb
Link layer: InfiniBand
[root@gz-cs-gpu-3-8 eden]# ibstatus
Infiniband device 'mlx4_0' port 1 status:
default gid: fe80:0000:0000:0000:0002:c903:0059:dddb
base lid: 0x3a
sm lid: 0x1
state: 4: ACTIVE
phys state: 5: LinkUp
rate: 40 Gb/sec (4X QDR)
link_layer: InfiniBand
| InfiniBand Link | Signal Pairs | Signaling Rate | Data Rate (Full Duplex) |
| 1X-SDR | 2 | 2.5 Gbps | 2.0 Gbps |
| 4X-SDR | 8 | 10 Gbps (4 x 2.5 Gbps) | 8 Gbps (4 x 2 Gbps) |
| 12X-SDR | 24 | 30 Gbps (12 x 2.5 Gbps) | 24 Gbps (12 x 2 Gbps) |
| 1X-DDR | 2 | 5 Gbps | 4.0 Gbps |
| 4X-DDR | 8 | 20 Gbps (4 x 5 Gbps) | 16 Gbps (4 x 4 Gbps) |
| 12X-DDR | 24 | 60 Gbps (12 x 5 Gbps) | 48 Gbps (12 x 4 Gbps) |
| 1X-QDR | 2 | 10 Gbps | 8.0 Gbps |
| 4X-QDR | 8 | 40 Gbps (4 x 5 Gbps) | 32 Gbps (4 x 8 Gbps) |
| 12XQDDR | 24 | 1200 Gbps (12 x 5 Gbps) | 96 Gbps (12 x 8 Gbps) |