主要想找到從nvme設備 到 vfio驅動的代碼路徑
[spdk/lib/nvme/nvme_pcie.c]
nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,bool direct_connect) (nvme_tcp,nvme_rdma等都有自己的scan,probe等函數，統一被封裝起來,如下)
1 int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, 2 bool direct_connect) 3 { 4 NVME_TRANSPORT_CALL(probe_ctx->trid.trtype, ctrlr_scan, (probe_ctx, direct_connect)); 5 }
1.nvme_pcie_ctrl_scan:
1 int 2 nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, 3 bool direct_connect) 4 { 5 .......//若probe_ctx中有指定nvme pci id，has_pci_addr=true 6 if (enum_ctx.has_pci_addr == false) { 7 return spdk_pci_enumerate(spdk_pci_nvme_get_driver(),//若沒有指定 8 pcie_nvme_enum_cb, &enum_ctx); 9 } else { 10 return spdk_pci_device_attach(spdk_pci_nvme_get_driver(),//指定了話直接attach 11 pcie_nvme_enum_cb, &enum_ctx, &enum_ctx.pci_addr); 12 } 13
spdk_pci_enumerate:
1 int 2 spdk_pci_enumerate(struct spdk_pci_driver *driver, 3 spdk_pci_enum_cb enum_cb, 4 void *enum_ctx) 5 { 6 struct spdk_pci_device *dev; 7 int rc; 8 9 cleanup_pci_devices(); 10 11 pthread_mutex_lock(&g_pci_mutex); 12 TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { //遍歷g_pci_devices 13 if (dev->internal.attached || 14 dev->internal.driver != driver || 15 dev->internal.pending_removal) { 16 continue; 17 } 18 19 rc = enum_cb(enum_ctx, dev);// 從g_pci_devices找到我們要的driver，那么調用回調函數進行后續的ctrl_construct工作 20 if (rc == 0) { 21 dev->internal.attached = true;//構建成功，attach設置為true 22 } else if (rc < 0) { 23 pthread_mutex_unlock(&g_pci_mutex); 24 return -1; 25 } 26 } 27 pthread_mutex_unlock(&g_pci_mutex); 28 29 if (!driver->is_registered) {//若改driver還沒注冊，那么register到rte_pci_bus.driver_list 30 driver->is_registered = true; 31 rte_pci_register(&driver->driver); 32 } 33 //上面不是找到了嗎，下面這些是干啥？為啥要scan和probe？？上面只是放到rte_pci_bus.driver_list了，后面還需做什么？ 34 driver->cb_fn = enum_cb; 35 driver->cb_arg = enum_ctx; 36 37 if (rte_bus_scan() != 0 || rte_bus_probe() != 0) {//若bus scan和probe都不成功 38 driver->cb_arg = NULL; 39 driver->cb_fn = NULL; 40 return -1; 41 } 42 43 driver->cb_arg = NULL; 44 driver->cb_fn = NULL; 45 46 cleanup_pci_devices(); 47 return 0; 48 }
回調函數pcie_nvme_enum_cb：
主要是執行nvme_ctrlr_probe(&trid, enum_ctx->probe_ctx, pci_dev) 從而構建相應type的ctrl：nvme_transport_ctrlr_construct(trid, &opts, devhandle)，這些不放在這里介紹，會在spdk的源碼解讀里面分析。
rte_pci_register:
1 /* register a driver */ 2 void 3 rte_pci_register(struct rte_pci_driver *driver) 4 { 5 TAILQ_INSERT_TAIL(&rte_pci_bus.driver_list, driver, next); 6 driver->bus = &rte_pci_bus; 7 }
//注意這個
struct rte_pci_bus rte_pci_bus = {
.bus = {
.scan = rte_pci_scan,
.probe = rte_pci_probe,
.find_device = pci_find_device,
.plug = pci_plug,
.unplug = pci_unplug,
.parse = pci_parse,
.dma_map = pci_dma_map,
.dma_unmap = pci_dma_unmap,
.get_iommu_class = rte_pci_get_iommu_class,
.dev_iterate = rte_pci_dev_iterate,
.hot_unplug_handler = pci_hot_unplug_handler,
.sigbus_handler = pci_sigbus_handler,
},
.device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list),
.driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list),
};
rte_bus_scan()和rte_bus_probe() 對所有bus進行scan和probe，若是pci_bus,執行的是上面注冊的
.scan = rte_pci_scan,
.probe = rte_pci_probe,
回到 1 中的attach函數：
spdk_pci_device_attach：跟原來看的有些不一樣，之前看的版本直接調用的是rte_pci_scan
1 int 2 spdk_pci_device_attach(struct spdk_pci_driver *driver, 3 spdk_pci_enum_cb enum_cb, 4 void *enum_ctx, struct spdk_pci_addr *pci_address) 5 { 6 struct spdk_pci_device *dev; 7 int rc; 8 char bdf[32]; 9 10 spdk_pci_addr_fmt(bdf, sizeof(bdf), pci_address); 11 12 cleanup_pci_devices(); 13 14 TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { 15 if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) { 16 break; 17 } 18 } 19 20 if (dev != NULL && dev->internal.driver == driver) { 21 pthread_mutex_lock(&g_pci_mutex); 22 if (dev->internal.attached || dev->internal.pending_removal) { 23 pthread_mutex_unlock(&g_pci_mutex); 24 return -1; 25 } 26 27 rc = enum_cb(enum_ctx, dev);//執行construct ctrl等 28 if (rc == 0) { 29 dev->internal.attached = true; 30 } 31 pthread_mutex_unlock(&g_pci_mutex); 32 return rc; 33 } 34 35 if (!driver->is_registered) { 36 driver->is_registered = true; 37 rte_pci_register(&driver->driver);//register到rte_pci_bus.driver_list 38 } 39 40 driver->cb_fn = enum_cb; 41 driver->cb_arg = enum_ctx; 42 43 #if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0) 44 int i = 0; 45 46 do { 47 rc = rte_eal_hotplug_add("pci", bdf, ""); 48 } while (rc == -ENOMSG && ++i <= DPDK_HOTPLUG_RETRY_COUNT); 49 50 if (i > 1 && rc == -EEXIST) { 51 /* Even though the previous request timed out, the device 52 * was attached successfully. 53 */ 54 rc = 0; 55 } 56 #else 57 rc = rte_eal_dev_attach(bdf, "");//這個函數在舊版本的dpdk中 58 #endif 59 60 driver->cb_arg = NULL; 61 driver->cb_fn = NULL; 62 63 cleanup_pci_devices(); 64 return rc == 0 ? 0 : -1; 65 }
rte_eal_hotplug_add:
--------rte_dev_probe(devargs)
1 int 2 rte_dev_probe(const char *devargs) 3 { 4 struct eal_dev_mp_req req; 5 struct rte_device *dev; 6 int ret; 7 8 memset(&req, 0, sizeof(req)); 9 req.t = EAL_DEV_REQ_TYPE_ATTACH; 10 strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN); 11 12 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {//不是primary進程 13 /** 14 * If in secondary process, just send IPC request to 15 * primary process. 16 */ 17 ret = eal_dev_hotplug_request_to_primary(&req);//那么給primary進程發attch的請求 18 if (ret != 0) { 19 RTE_LOG(ERR, EAL, 20 "Failed to send hotplug request to primary\n"); 21 return -ENOMSG; 22 } 23 if (req.result != 0) 24 RTE_LOG(ERR, EAL, 25 "Failed to hotplug add device\n"); 26 return req.result; 27 } 28 29 /* attach a shared device from primary start from here: */ 31 /* primary attach the new device itself. */ 32 ret = local_dev_probe(devargs, &dev);//其中會執行da->bus->scan() 33 34 if (ret != 0) { 35 RTE_LOG(ERR, EAL, 36 "Failed to attach device on primary process\n"); 37 38 /** 39 * it is possible that secondary process failed to attached a 40 * device that primary process have during initialization, 41 * so for -EEXIST case, we still need to sync with secondary 42 * process. 43 */ 44 if (ret != -EEXIST) 45 return ret; 46 } 47 48 /* primary send attach sync request to secondary. */ 49 ret = eal_dev_hotplug_request_to_secondary(&req);//給secondary進程發消息同步attach情況 50 51 /* if any communication error, we need to rollback. */ 52 if (ret != 0) { 53 RTE_LOG(ERR, EAL, 54 "Failed to send hotplug add request to secondary\n"); 55 ret = -ENOMSG; 56 goto rollback; 57 } 59 /** 60 * if any secondary failed to attach, we need to consider if rollback 61 * is necessary. 62 */ 63 if (req.result != 0) { 64 RTE_LOG(ERR, EAL, 65 "Failed to attach device on secondary process\n"); 66 ret = req.result; 67 68 /* for -EEXIST, we don't need to rollback. */ 69 if (ret == -EEXIST) 70 return ret; 71 goto rollback; 72 } 73 74 return 0; 75 76 rollback: 77 req.t = EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK; 78 79 /* primary send rollback request to secondary. */ 80 if (eal_dev_hotplug_request_to_secondary(&req) != 0) 81 RTE_LOG(WARNING, EAL, 82 "Failed to rollback device attach on secondary." 83 "Devices in secondary may not sync with primary\n"); 84 85 /* primary rollback itself. */ 86 if (local_dev_remove(dev) != 0) 87 RTE_LOG(WARNING, EAL, 88 "Failed to rollback device attach on primary." 89 "Devices in secondary may not sync with primary\n"); 90 91 return ret; 92 }
local_dev_probe:
最終也是調用rte_pci_scan和pci_probe_all_drivers
1 /* probe device at local process. */ 2 int 3 local_dev_probe(const char *devargs, struct rte_device **new_dev) 4 { 5 struct rte_device *dev; 6 struct rte_devargs *da; 7 int ret; 8 9 *new_dev = NULL; 10 da = calloc(1, sizeof(*da)); 11 if (da == NULL) 12 return -ENOMEM; 13 14 ret = rte_devargs_parse(da, devargs); 15 if (ret) 16 goto err_devarg; 17 18 if (da->bus->plug == NULL) { 19 RTE_LOG(ERR, EAL, "Function plug not supported by bus (%s)\n", 20 da->bus->name); 21 ret = -ENOTSUP; 22 goto err_devarg; 23 } 24 25 ret = rte_devargs_insert(&da); 26 if (ret) 27 goto err_devarg; 28 29 /* the rte_devargs will be referenced in the matching rte_device */ 30 ret = da->bus->scan();//pci的話執行register函數中注冊的rte_pci_scan 31 if (ret) 32 goto err_devarg; 33 34 dev = da->bus->find_device(NULL, cmp_dev_name, da->name); 35 if (dev == NULL) { 36 RTE_LOG(ERR, EAL, "Cannot find device (%s)\n", 37 da->name); 38 ret = -ENODEV; 39 goto err_devarg; 40 } 41 /* Since there is a matching device, it is now its responsibility 42 * to manage the devargs we've just inserted. From this point 43 * those devargs shouldn't be removed manually anymore. 44 */ 45 46 ret = dev->bus->plug(dev);//rte_pci_plug中執行的是pci_probe_all_drivers(RTE_DEV_TO_PCI(dev)) 47 if (ret > 0) 48 ret = -ENOTSUP; 49 50 if (ret && !rte_dev_is_probed(dev)) { /* if hasn't ever succeeded */ 51 RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n", 52 dev->name); 53 return ret; 54 } 55 56 *new_dev = dev; 57 return ret; 58 59 err_devarg: 60 if (rte_devargs_remove(da) != 0) { 61 free(da->args); 62 free(da); 63 } 64 return ret; 65 }
所以 1 中 nvme_pcie_ctrl_scan 不管是spdk_pci_enumerate還是spdk_pci_device_attach，核心的流程都是一樣的：
執行回調函數pcie_nvme_enum_cb；
rte_pci_scan
rte_pci_probe/pci_probe_all_drivers (rte_pci_probe也是調用pci_probe_all_drivers)
所以后面我們看 rte_pci_bus.scan（rte_pci_scan）和rte_pci_bus.probe（rte_pci_probe）的實現。
rte_pci_scan:
1 /* 2 * Scan the content of the PCI bus, and the devices in the devices 3 * list 4 */ 5 int 6 rte_pci_scan(void) 7 { 8 struct dirent *e; 9 DIR *dir; 10 char dirname[PATH_MAX]; 11 struct rte_pci_addr addr; 12 13 ........ 14 dir = opendir(rte_pci_get_sysfs_path()); ///sys/bus/pci/devices 15 16 ........ 17 while ((e = readdir(dir)) != NULL) { 18 if (e->d_name[0] == '.') 19 continue; 20 21 if (parse_pci_addr_format(e->d_name, sizeof(e->d_name), &addr) != 0) 22 continue; 23 24 snprintf(dirname, sizeof(dirname), "%s/%s", 25 rte_pci_get_sysfs_path(), e->d_name); 26 27 if (pci_scan_one(dirname, &addr) < 0)//掃描/sys/bus/pci/devices下面每個目錄 28 goto error; 29 } 30 closedir(dir); 31 return 0; 32 .......... 33 }
ls /sys/bus/pci/devices/
0000:00:00.0 0000:00:04.4 0000:00:1c.6 0000:03:00.0 0000:ff:0d.1 0000:ff:12.2 0000:ff:14.5 0000:ff:17.6
0000:00:01.0 0000:00:04.5 0000:00:1c.7 0000:05:00.0 0000:ff:0f.0 0000:ff:12.4 0000:ff:14.6 0000:ff:17.7
1 /* Scan one pci sysfs entry, and fill the devices list from it. */ 2 static int 3 pci_scan_one(const char *dirname, const struct rte_pci_addr *addr) 4 { 5 char filename[PATH_MAX]; 6 unsigned long tmp; 7 struct rte_pci_device *dev; 8 char driver[PATH_MAX]; 9 int ret; 10 11 dev = malloc(sizeof(*dev)); 12 if (dev == NULL) 13 return -1; 14 15 memset(dev, 0, sizeof(*dev)); 16 dev->device.bus = &rte_pci_bus.bus; 17 dev->addr = *addr; 18 19 /* get vendor id */ 20 snprintf(filename, sizeof(filename), "%s/vendor", dirname); 21 if (eal_parse_sysfs_value(filename, &tmp) < 0) { 22 free(dev); 23 return -1; 24 } 25 dev->id.vendor_id = (uint16_t)tmp; 26 27 /* get device id */ 28 snprintf(filename, sizeof(filename), "%s/device", dirname); 29 if (eal_parse_sysfs_value(filename, &tmp) < 0) { 30 free(dev); 31 return -1; 32 } 33 dev->id.device_id = (uint16_t)tmp; 34 35 /* get subsystem_vendor id */ 36 snprintf(filename, sizeof(filename), "%s/subsystem_vendor", 37 dirname); 38 if (eal_parse_sysfs_value(filename, &tmp) < 0) { 39 free(dev); 40 return -1; 41 } 42 dev->id.subsystem_vendor_id = (uint16_t)tmp; 43 44 /* get subsystem_device id */ 45 snprintf(filename, sizeof(filename), "%s/subsystem_device", 46 dirname); 47 if (eal_parse_sysfs_value(filename, &tmp) < 0) { 48 free(dev); 49 return -1; 50 } 51 dev->id.subsystem_device_id = (uint16_t)tmp; 52 53 /* get class_id */ 54 snprintf(filename, sizeof(filename), "%s/class", 55 dirname); 56 if (eal_parse_sysfs_value(filename, &tmp) < 0) { 57 free(dev); 58 return -1; 59 } 60 /* the least 24 bits are valid: class, subclass, program interface */ 61 dev->id.class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID; 62 63 /* get max_vfs */ 64 dev->max_vfs = 0; 65 snprintf(filename, sizeof(filename), "%s/max_vfs", dirname); 66 if (!access(filename, F_OK) && 67 eal_parse_sysfs_value(filename, &tmp) == 0) 68 dev->max_vfs = (uint16_t)tmp; 69 else { 70 /* for non igb_uio driver, need kernel version >= 3.8 */ 71 snprintf(filename, sizeof(filename), 72 "%s/sriov_numvfs", dirname); 73 if (!access(filename, F_OK) && 74 eal_parse_sysfs_value(filename, &tmp) == 0) 75 dev->max_vfs = (uint16_t)tmp; 76 } 77 78 /* get numa node, default to 0 if not present */ 79 snprintf(filename, sizeof(filename), "%s/numa_node", 80 dirname); 81 82 if (access(filename, F_OK) != -1) { 83 if (eal_parse_sysfs_value(filename, &tmp) == 0) 84 dev->device.numa_node = tmp; 85 else 86 dev->device.numa_node = -1; 87 } else { 88 dev->device.numa_node = 0; 89 } 90 91 pci_name_set(dev); 92 93 /* parse resources */ 94 snprintf(filename, sizeof(filename), "%s/resource", dirname); 95 if (pci_parse_sysfs_resource(filename, dev) < 0) { 96 RTE_LOG(ERR, EAL, "%s(): cannot parse resource\n", __func__); 97 free(dev); 98 return -1; 99 } 100 101 /* parse driver */ 102 snprintf(filename, sizeof(filename), "%s/driver", dirname);///sys/bus/pci/drivers 103 ret = pci_get_kernel_driver_by_path(filename, driver, sizeof(driver)); 104 if (ret < 0) { 105 RTE_LOG(ERR, EAL, "Fail to get kernel driver\n"); 106 free(dev); 107 return -1; 108 } 109 110 if (!ret) { 111 if (!strcmp(driver, "vfio-pci")) 112 dev->kdrv = RTE_KDRV_VFIO; 113 else if (!strcmp(driver, "igb_uio")) 114 dev->kdrv = RTE_KDRV_IGB_UIO; 115 else if (!strcmp(driver, "uio_pci_generic")) 116 dev->kdrv = RTE_KDRV_UIO_GENERIC; 117 else 118 dev->kdrv = RTE_KDRV_UNKNOWN; 119 } else 120 dev->kdrv = RTE_KDRV_NONE; 121 122 /* device is valid, add in list (sorted) */ 123 if (TAILQ_EMPTY(&rte_pci_bus.device_list)) { 124 rte_pci_add_device(dev); 125 } else { 126 struct rte_pci_device *dev2; 127 int ret; 128 129 TAILQ_FOREACH(dev2, &rte_pci_bus.device_list, next) { 130 ret = rte_pci_addr_cmp(&dev->addr, &dev2->addr); 131 if (ret > 0) 132 continue; 133 134 if (ret < 0) { 135 rte_pci_insert_device(dev2, dev); 136 } else { /* already registered */ 137 if (!rte_dev_is_probed(&dev2->device)) { 138 dev2->kdrv = dev->kdrv; 139 dev2->max_vfs = dev->max_vfs; 140 pci_name_set(dev2); 141 memmove(dev2->mem_resource, 142 dev->mem_resource, 143 sizeof(dev->mem_resource)); 144 } else { 145 /** 146 * If device is plugged and driver is 147 * probed already, (This happens when 148 * we call rte_dev_probe which will 149 * scan all device on the bus) we don't 150 * need to do anything here unless... 151 **/ 152 if (dev2->kdrv != dev->kdrv || 153 dev2->max_vfs != dev->max_vfs) 154 /* 155 * This should not happens. 156 * But it is still possible if 157 * we unbind a device from 158 * vfio or uio before hotplug 159 * remove and rebind it with 160 * a different configure. 161 * So we just print out the 162 * error as an alarm. 163 */ 164 RTE_LOG(ERR, EAL, "Unexpected device scan at %s!\n", 165 filename); 166 } 167 free(dev); 168 } 169 return 0; 170 } 171 172 rte_pci_add_device(dev); 173 } 174 175 return 0; 176 }
rte_pci_probe:
------FOREACH_DEVICE_ON_PCIBUS(dev) pci_probe_all_drivers(dev)
------------FOREACH_DRIVER_ON_PCIBUS(dr) rte_pci_probe_one_driver(dr, dev)
1 /* 2 * If vendor/device ID match, call the probe() function of the 3 * driver. 4 */ 5 static int 6 rte_pci_probe_one_driver(struct rte_pci_driver *dr, 7 struct rte_pci_device *dev) 8 { 9 int ret; 10 bool already_probed; 11 struct rte_pci_addr *loc; 12 13 if ((dr == NULL) || (dev == NULL)) 14 return -EINVAL; 15 16 loc = &dev->addr; 17 18 /* The device is not blacklisted; Check if driver supports it */ 19 if (!rte_pci_match(dr, dev)) 20 /* Match of device and driver failed */ 21 return 1; 22 23 RTE_LOG(INFO, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n", 24 loc->domain, loc->bus, loc->devid, loc->function, 25 dev->device.numa_node); 26 27 /* no initialization when blacklisted, return without error */ 28 if (dev->device.devargs != NULL && 29 dev->device.devargs->policy == 30 RTE_DEV_BLACKLISTED) { 31 RTE_LOG(INFO, EAL, " Device is blacklisted, not" 32 " initializing\n"); 33 return 1; 34 } 35 36 if (dev->device.numa_node < 0) { 37 RTE_LOG(WARNING, EAL, " Invalid NUMA socket, default to 0\n"); 38 dev->device.numa_node = 0; 39 } 40 41 already_probed = rte_dev_is_probed(&dev->device); 42 if (already_probed && !(dr->drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 43 RTE_LOG(DEBUG, EAL, "Device %s is already probed\n", 44 dev->device.name); 45 return -EEXIST; 46 } 47 48 RTE_LOG(INFO, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id, 49 dev->id.device_id, dr->driver.name); 50 51 /* 52 * reference driver structure 53 * This needs to be before rte_pci_map_device(), as it enables to use 54 * driver flags for adjusting configuration. 55 */ 56 if (!already_probed) { 57 enum rte_iova_mode dev_iova_mode; 58 enum rte_iova_mode iova_mode; 59 60 dev_iova_mode = pci_device_iova_mode(dr, dev);//從設備綁定的驅動判斷dev_iova_mode,例如若是UIO驅動，則這里是PA 61 iova_mode = rte_eal_iova_mode();//dpdk初始化過程判斷的，首先會去判斷設備的驅動，如果驅動還沒注冊，那么里面是根據機器上是否有iommu等判斷。而spdk中nvme
驅動是在dpdk初始化之后才注冊進去的，所以即使是用uio驅動，dpdk給出的iova_mode也是VA，所以這里有點小問題！
62 if (dev_iova_mode != RTE_IOVA_DC && 63 dev_iova_mode != iova_mode) { 64 RTE_LOG(ERR, EAL, " Expecting '%s' IOVA mode but current mode is '%s', not initializing\n", 65 dev_iova_mode == RTE_IOVA_PA ? "PA" : "VA", 66 iova_mode == RTE_IOVA_PA ? "PA" : "VA"); 67 return -EINVAL; 68 } 69 70 dev->driver = dr; 71 } 72 73 if (!already_probed && (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING)) { 74 /* map resources for devices that use igb_uio */？？？注釋有問題?? 75 ret = rte_pci_map_device(dev); 76 if (ret != 0) { 77 dev->driver = NULL; 78 return ret; 79 } 80 } 81 82 /* call the driver probe() function */ 83 ret = dr->probe(dr, dev); //在哪里？？？？？ 84 if (already_probed) 85 return ret; /* no rollback if already succeeded earlier */ 86 if (ret) { 87 dev->driver = NULL; 88 if ((dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) && 89 /* Don't unmap if device is unsupported and 90 * driver needs mapped resources. 91 */ 92 !(ret > 0 && 93 (dr->drv_flags & RTE_PCI_DRV_KEEP_MAPPED_RES))) 94 rte_pci_unmap_device(dev); 95 } else { 96 dev->device.driver = &dr->driver; 97 } 98 99 return ret; 100 }
rte_pci_map_device 從這遍開始和vfio，uio等相關
1 /* Map pci device */ 2 int 3 rte_pci_map_device(struct rte_pci_device *dev) 4 { 5 int ret = -1; 6 7 /* try mapping the NIC resources using VFIO if it exists */ 8 switch (dev->kdrv) { 9 case RTE_KDRV_VFIO: 10 #ifdef VFIO_PRESENT 11 if (pci_vfio_is_enabled()) 12 ret = pci_vfio_map_resource(dev); 13 #endif 14 break; 15 case RTE_KDRV_IGB_UIO: 16 case RTE_KDRV_UIO_GENERIC: 17 if (rte_eal_using_phys_addrs()) { 18 /* map resources for devices that use uio */ 19 ret = pci_uio_map_resource(dev); 20 } 21 break; 22 default: 23 RTE_LOG(DEBUG, EAL, 24 " Not managed by a supported kernel driver, skipped\n"); 25 ret = 1; 26 break; 27 } 28 29 return ret; 30 }
pci_vfio_map_resource:
---1---pci_vfio_map_resource_primary 若設primary進程
------pci_vfio_map_resource_secondary
-------2---------rte_vfio_setup_device
------------3------------vfio_mem_event_callback
這三個函數要重點看。。。。