在鯤鵬916服務器上編譯和安裝dpdk踩坑

 

 無法爬樓樓的可以從這里下：

http://static.dpdk.org/rel/

 

[root@kunpeng82 data1]#  cat  /etc/default/grub |grep GRUB_CMDLINE_LINUX
GRUB_CMDLINE_LINUX="crashkernel=auto"
GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=ttyAMA0,115200 no_timer_check nofb nomodeset vga=normal"
[root@kunpeng82 data1]# mkdir /mnt/huge
[root@kunpeng82 data1]# mount -t hugetlbfs nodev /mnt/huge
[root@kunpeng82 data1]# grep Hugepagesize /proc/meminfo
Hugepagesize:     524288 kB

[root@kunpeng82 devuser]# ls /sys/kernel/mm/hugepages/
hugepages-2048kB  hugepages-524288kB
[root@kunpeng82 devuser]#

[root@kunpeng82 devuser]# ls /sys/kernel/mm/hugepages/hugepages-524288kB/
free_hugepages nr_hugepages nr_hugepages_mempolicy nr_overcommit_hugepages resv_hugepages surplus_hugepages
[root@kunpeng82 devuser]# cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
0
[root@kunpeng82 devuser]#

[root@kunpeng82 devuser]# echo 256 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@kunpeng82 devuser]# cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
256
[root@kunpeng82 devuser]#

[root@kunpeng82 devuser]# cat /proc/meminfo |grep HugePages_
HugePages_Total: 256
HugePages_Free: 256
HugePages_Rsvd: 0
HugePages_Surp: 0
[root@kunpeng82 devuser]#

 

配置DPDK編譯目標

DPDK支持多種平台（x86、x64、ARM），編譯的目標由ARCH-MACHINE-EXECENV-TOOLCHAIN給出

ARCH can be: i686, x86_64, ppc_64, arm64
MACHINE can be: native, power8, armv8a
EXECENV can be: linuxapp,bsdapp
TOOLCHAIN can be: gcc, icc

[root@kunpeng82 data1]# lscpu
Architecture:          aarch64
Byte Order:            Little Endian
CPU(s):                64
On-line CPU(s) list:   0-63
Thread(s) per core:    1
Core(s) per socket:    32
Socket(s):             2
NUMA node(s):          4
Model:                 2
BogoMIPS:              100.00
L1d cache:             32K
L1i cache:             48K
L2 cache:              1024K
L3 cache:              16384K
NUMA node0 CPU(s):     0-15
NUMA node1 CPU(s):     16-31
NUMA node2 CPU(s):     32-47
NUMA node3 CPU(s):     48-63
Flags:                 fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid

export RTE_TARGET=arm64-armv8a-linuxapp-gcc
export RTE_SDK=/data1/dpdk_sdk

make config T=arm64-armv8a-linuxapp-gcc
make -j 64
make install T=arm64-armv8a-linuxapp-gcc  -j 64

 

 

 

[root@kunpeng82 dpdk-19.11]# make
== Build lib
== Build lib/librte_kvargs
  SYMLINK-FILE include/rte_kvargs.h
  CC rte_kvargs.o
  AR librte_kvargs.a
  INSTALL-LIB librte_kvargs.a
== Build lib/librte_eal
== Build lib/librte_eal/common
  SYMLINK-FILE include/rte_branch_prediction.h
  SYMLINK-FILE include/rte_common.h
  SYMLINK-FILE include/rte_compat.h
  SYMLINK-FILE include/rte_function_versioning.h
  SYMLINK-FILE include/rte_debug.h
  SYMLINK-FILE include/rte_eal.h
  SYMLINK-FILE include/rte_eal_interrupts.h
  SYMLINK-FILE include/rte_errno.h
  SYMLINK-FILE include/rte_launch.h
  SYMLINK-FILE include/rte_lcore.h
  SYMLINK-FILE include/rte_log.h
  SYMLINK-FILE include/rte_memory.h
  SYMLINK-FILE include/rte_memzone.h
  SYMLINK-FILE include/rte_per_lcore.h
  SYMLINK-FILE include/rte_random.h
  SYMLINK-FILE include/rte_tailq.h
  SYMLINK-FILE include/rte_interrupts.h
  SYMLINK-FILE include/rte_alarm.h
  SYMLINK-FILE include/rte_string_fns.h
  SYMLINK-FILE include/rte_version.h
  SYMLINK-FILE include/rte_eal_memconfig.h
  SYMLINK-FILE include/rte_hexdump.h
  SYMLINK-FILE include/rte_devargs.h
  SYMLINK-FILE include/rte_bus.h
  SYMLINK-FILE include/rte_dev.h
  SYMLINK-FILE include/rte_class.h
  SYMLINK-FILE include/rte_option.h
  SYMLINK-FILE include/rte_pci_dev_feature_defs.h
  SYMLINK-FILE include/rte_pci_dev_features.h
  SYMLINK-FILE include/rte_malloc.h
  SYMLINK-FILE include/rte_keepalive.h
  SYMLINK-FILE include/rte_time.h
  SYMLINK-FILE include/rte_service.h
  SYMLINK-FILE include/rte_service_component.h
  SYMLINK-FILE include/rte_bitmap.h
  SYMLINK-FILE include/rte_vfio.h
  SYMLINK-FILE include/rte_hypervisor.h
  SYMLINK-FILE include/rte_test.h
  SYMLINK-FILE include/rte_reciprocal.h
  SYMLINK-FILE include/rte_fbarray.h
  SYMLINK-FILE include/rte_uuid.h
  SYMLINK-FILE include/rte_atomic.h
  SYMLINK-FILE include/rte_atomic_32.h
  SYMLINK-FILE include/rte_atomic_64.h
  SYMLINK-FILE include/rte_byteorder.h
  SYMLINK-FILE include/rte_cpuflags.h
  SYMLINK-FILE include/rte_cpuflags_32.h
  SYMLINK-FILE include/rte_cpuflags_64.h
  SYMLINK-FILE include/rte_cycles.h
  SYMLINK-FILE include/rte_cycles_32.h
  SYMLINK-FILE include/rte_cycles_64.h
  SYMLINK-FILE include/rte_io.h
  SYMLINK-FILE include/rte_io_64.h
  SYMLINK-FILE include/rte_mcslock.h
  SYMLINK-FILE include/rte_memcpy.h
  SYMLINK-FILE include/rte_memcpy_32.h
  SYMLINK-FILE include/rte_memcpy_64.h
  SYMLINK-FILE include/rte_pause.h
  SYMLINK-FILE include/rte_pause_32.h
  SYMLINK-FILE include/rte_pause_64.h
  SYMLINK-FILE include/rte_prefetch.h
  SYMLINK-FILE include/rte_prefetch_32.h
  SYMLINK-FILE include/rte_prefetch_64.h
  SYMLINK-FILE include/rte_rwlock.h
  SYMLINK-FILE include/rte_spinlock.h
  SYMLINK-FILE include/rte_ticketlock.h
  SYMLINK-FILE include/rte_vect.h
  SYMLINK-FILE include/generic/rte_atomic.h
  SYMLINK-FILE include/generic/rte_byteorder.h
  SYMLINK-FILE include/generic/rte_cycles.h
  SYMLINK-FILE include/generic/rte_prefetch.h
  SYMLINK-FILE include/generic/rte_memcpy.h
  SYMLINK-FILE include/generic/rte_cpuflags.h
  SYMLINK-FILE include/generic/rte_mcslock.h
  SYMLINK-FILE include/generic/rte_spinlock.h
  SYMLINK-FILE include/generic/rte_rwlock.h
  SYMLINK-FILE include/generic/rte_ticketlock.h
  SYMLINK-FILE include/generic/rte_vect.h
  SYMLINK-FILE include/generic/rte_pause.h
  SYMLINK-FILE include/generic/rte_io.h
== Build lib/librte_eal/linux
== Build lib/librte_eal/linux/eal
  SYMLINK-FILE include/rte_kni_common.h
  SYMLINK-FILE include/rte_os.h
  CC eal.o
  CC eal_cpuflags.o
  CC eal_hugepage_info.o
  CC eal_memory.o
/data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_memory.c:32:18: fatal error: numa.h: No such file or directory
 #include <numa.h>
                  ^
compilation terminated.
make[5]: *** [eal_memory.o] Error 1
make[4]: *** [eal] Error 2
make[3]: *** [linux] Error 2
make[2]: *** [librte_eal] Error 2
make[1]: *** [lib] Error 2
make: *** [all] Error 2
[root@kunpeng82 dpdk-19.11]# 

dpdk編譯錯誤 numa.h

libnuma-dev的安裝：yum install numactl libnuma-dev

安裝
yum install numactl-devel*

 

make: *** /lib/modules/4.14.0-115.10.1.el7a.aarch64/build: No such file or directory.  Stop

[root@kunpeng82 dpdk-19.11]# ls /lib/modules/4.14.0-115.10.1.el7a.aarch64/build 
/lib/modules/4.14.0-115.10.1.el7a.aarch64/build

 

 查看鏈接狀態，執行以上指令，可以看到build鏈接到了一個不存在的路徑

 

[root@kunpeng82 dpdk-19.11]# rm -rfv  /lib/modules/4.14.0-115.10.1.el7a.aarch64/build 
removed ‘/lib/modules/4.14.0-115.10.1.el7a.aarch64/build’

[root@kunpeng82 dpdk-19.11]# ls /usr/src/kernels/
4.18.0-147.0.3.el7.aarch64  4.18.0-147.0.3.el7.aarch64+debug
[root@kunpeng82 dpdk-19.11]# ls /usr/src/kernels/4.18.0-147.0.3.el7.aarch64
arch   crypto    fs       ipc      lib               mm              samples   sound       usr
block  drivers   include  Kconfig  Makefile          Module.symvers  scripts   System.map  virt
certs  firmware  init     kernel   Makefile.rhelver  net             security  tools       vmlinux.id
[root@kunpeng82 dpdk-19.11]# ln -s /usr/src/kernels/4.18.0-147.0.3.el7.aarch64  /lib/modules/4.14.0-115.10.1.el7a.aarch64/build
[root@kunpeng82 dpdk-19.11]# 

 

 

編譯DPDK的時候可以不用使用系統環境的編譯
export RTE_KERNELDIR=$PWD

[root@kunpeng82 dpdk-19.11]# yum install kernel-devel*
Loaded plugins: fastestmirror
Repository base is listed more than once in the configuration
Repository updates is listed more than once in the configuration
Repository extras is listed more than once in the configuration
Repository centosplus is listed more than once in the configuration
Loading mirror speeds from cached hostfile
Package kernel-devel-4.18.0-147.0.3.el7.aarch64 already installed and latest version
Nothing to do
[root@kunpeng82 dpdk-19.11]# yum install kernel-headers*
Loaded plugins: fastestmirror
Repository base is listed more than once in the configuration
Repository updates is listed more than once in the configuration
Repository extras is listed more than once in the configuration
Repository centosplus is listed more than once in the configuration
Loading mirror speeds from cached hostfile
Package kernel-headers-4.18.0-147.0.3.el7.aarch64 already installed and latest version
Nothing to do
[root@kunpeng82 dpdk-19.11]#

 

[root@kunpeng82 dpdk-19.11]# modprobe vfio-pci
[root@kunpeng82 dpdk-19.11]# modprobe vfio
[root@kunpeng82 dpdk-19.11]# ls

一個設備都沒有檢測到
[root@kunpeng82 dpdk-19.11]# ./usertools/dpdk-devbind.py  -s

No 'Network' devices detected
=============================

No 'Baseband' devices detected
==============================

No 'Crypto' devices detected
============================

No 'Eventdev' devices detected
==============================

No 'Mempool' devices detected
=============================

No 'Compress' devices detected
==============================

No 'Misc (rawdev)' devices detected
===================================
[root@kunpeng82 dpdk-19.11]# 

 

 總結：

    鯤鵬916上網卡不支持dpdk

 安裝依賴包（如果已安裝，可以忽略）

安裝依賴包（如果已安裝，可以忽略）

yum install kernel-devel

yum install numactl-devel

配置環境變量：

export RTE_SDK=/home/dpdk-19.11 (以當前環境DPDK源碼實際路徑為准)

export RTE_TARGET= arm64-armv8a-linuxapp-gcc (以當前環境CPU架構為准)

在當前DPDK源碼目錄下編譯DPDK：

    make install T=$RTE_TARGET -j 48

 

 

debug

export RTE_SDK=/data1/dpdk-19.11
export RTE_TARGET=arm64-armv8a-linuxapp-gcc
//export CFLAGS+="-O0 -g"
export EXTRA_CFLAGS="-O0 -g"
make install   T=$RTE_TARGET -j 128
# ingore Installation cannot run with T defined and DESTDIR undefined

 

 

 

 

 

=====================================================

[root@localhost dpdk-19.11]# mkdir /mnt/huge

[root@localhost dpdk-19.11]# mount -t hugetlbfs nodev /mnt/huge
[root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# ls /sys/kernel/mm/hugepages/
hugepages-2048kB hugepages-524288kB
[root@localhost dpdk-19.11]# grep Hugepagesize /proc/meminfo
Hugepagesize: 524288 kB
[root@localhost dpdk-19.11]

cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
echo 256 /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages

[root@localhost dpdk-19.11]# export RTE_TARGET= arm64-armv8a-linuxapp-gcc
-bash: export: `arm64-armv8a-linuxapp-gcc': not a valid identifier
[root@localhost dpdk-19.11]# export RTE_TARGET=arm64-armv8a-linuxapp-gcc
[root@localhost dpdk-19.11]#
yum install -y *gcc* *glibc* *stdc* *gfortran* m4 numactl* automake
[root@localhost dpdk-19.11]# ls /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/
ls: cannot access /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/: No such file or directory
 

 

[root@localhost data1]# ls
CentOS-7-aarch64-Everything-1810.iso  dpdk-19.11.kunpeng.tgz  lost+found
dpdk-19.11                            hicn-master.zip         Python-3.7.7.tgz
[root@localhost data1]# mkdir -p /mnt/CentOS-7.6
[root@localhost data1]# mount -o loop  CentOS-7-aarch64-Everything-1810.iso  /mnt/CentOS-7.6
mount: /dev/loop0 is write-protected, mounting read-only
[root@localhost data1]# vi /etc/yum.
yum.conf     yum.repos.d/ 
[root@localhost data1]# vi /etc/yum.repos.d/local.repo 
[local]
name=local.repo
baseurl=file:///mnt/CentOS-7.6
enabled=1
gpgcheck=0

備份yum.repos.d下的的其他repo

 

 

[root@localhost data1]# yum makecache
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
local                                                               | 3.6 kB  00:00:00     
Metadata Cache Created
[root@localhost data1]# yum list | grep kernel-dev
kernel-devel.aarch64                        4.14.0-115.el7a.0.1        local    
[root@localhost data1]# 

 

[root@localhost dpdk-19.11]# ls /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/ 
arch    drivers   init     lib             net       sound       virt
block   firmware  ipc      Makefile        samples   System.map  vmlinux.id
certs   fs        Kconfig  mm              scripts   tools
crypto  include   kernel   Module.symvers  security  usr
[root@localhost dpdk-19.11]#