Again, a couple of directly related functions at once get adjusted to account for the segment number. Can we re-define bit assignments for the four uses of machine_bdf in the domctl interface? Signed-off-by: Jan Beulich --- 2011-08-25.orig/tools/libxl/libxl_pci.c 2011-07-25 09:10:24.000000000 +0200 +++ 2011-08-25/tools/libxl/libxl_pci.c 2011-08-25 15:06:40.000000000 +0200 @@ -45,7 +45,7 @@ static unsigned int pcidev_encode_bdf(li { unsigned int value; - value = 0; + value = pcidev->domain << 24; /* XXX losing to 8 bits */ value |= (pcidev->bus & 0xff) << 16; value |= (pcidev->dev & 0x1f) << (8+3); value |= (pcidev->func & 0x7) << (8+0); --- 2011-08-25.orig/tools/python/xen/lowlevel/xc/xc.c 2010-11-05 09:22:58.000000000 +0100 +++ 2011-08-25/tools/python/xen/lowlevel/xc/xc.c 2011-08-25 15:06:40.000000000 +0200 @@ -619,6 +619,7 @@ static PyObject *pyxc_test_assign_device while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) ) { + bdf = seg << 24; /* XXX losing top 8 bits */ bdf |= (bus & 0xff) << 16; bdf |= (dev & 0x1f) << 11; bdf |= (func & 0x7) << 8; @@ -651,6 +652,7 @@ static PyObject *pyxc_assign_device(XcOb while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) ) { + bdf = seg << 24; /* XXX losing top 8 bits */ bdf |= (bus & 0xff) << 16; bdf |= (dev & 0x1f) << 11; bdf |= (func & 0x7) << 8; @@ -683,6 +685,7 @@ static PyObject *pyxc_deassign_device(Xc while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) ) { + bdf = seg << 24; /* XXX losing top 8 bits */ bdf |= (bus & 0xff) << 16; bdf |= (dev & 0x1f) << 11; bdf |= (func & 0x7) << 8; @@ -720,6 +723,7 @@ static PyObject *pyxc_get_device_group(X if (sdev_array == NULL) return PyErr_NoMemory(); + bdf = seg << 24; /* XXX losing top 8 bits */ bdf |= (bus & 0xff) << 16; bdf |= (dev & 0x1f) << 11; bdf |= (func & 0x7) << 8; --- 2011-08-25.orig/xen/arch/x86/domctl.c 2011-06-16 09:21:02.000000000 +0200 +++ 2011-08-25/xen/arch/x86/domctl.c 2011-08-25 15:06:40.000000000 +0200 @@ -746,6 +746,7 @@ long arch_do_domctl( { struct domain *d; u32 max_sdevs; + u16 seg; u8 bus, devfn; XEN_GUEST_HANDLE_64(uint32) sdevs; int num_sdevs; @@ -758,12 +759,14 @@ long arch_do_domctl( if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL ) break; + seg = domctl->u.get_device_group.machine_bdf >> 24; bus = (domctl->u.get_device_group.machine_bdf >> 16) & 0xff; devfn = (domctl->u.get_device_group.machine_bdf >> 8) & 0xff; max_sdevs = domctl->u.get_device_group.max_sdevs; sdevs = domctl->u.get_device_group.sdev_array; - num_sdevs = iommu_get_device_group(d, bus, devfn, sdevs, max_sdevs); + num_sdevs = iommu_get_device_group(d, seg, bus, devfn, + sdevs, max_sdevs); if ( num_sdevs < 0 ) { dprintk(XENLOG_ERR, "iommu_get_device_group() failed!\n"); @@ -783,6 +786,7 @@ long arch_do_domctl( case XEN_DOMCTL_test_assign_device: { + u16 seg; u8 bus, devfn; ret = -ENOSYS; @@ -794,14 +798,15 @@ long arch_do_domctl( break; ret = -EINVAL; + seg = domctl->u.get_device_group.machine_bdf >> 24; bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff; devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff; - if ( device_assigned(bus, devfn) ) + if ( device_assigned(seg, bus, devfn) ) { gdprintk(XENLOG_ERR, "XEN_DOMCTL_test_assign_device: " - "%x:%x.%x already assigned, or non-existent\n", - bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + "%04x:%02x:%02x.%u already assigned, or non-existent\n", + seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); break; } ret = 0; @@ -811,6 +816,7 @@ long arch_do_domctl( case XEN_DOMCTL_assign_device: { struct domain *d; + u16 seg; u8 bus, devfn; ret = -ENOSYS; @@ -829,14 +835,15 @@ long arch_do_domctl( if ( ret ) goto assign_device_out; + seg = domctl->u.get_device_group.machine_bdf >> 24; bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff; devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff; - ret = assign_device(d, bus, devfn); + ret = assign_device(d, seg, bus, devfn); if ( ret ) gdprintk(XENLOG_ERR, "XEN_DOMCTL_assign_device: " - "assign device (%x:%x.%x) failed\n", - bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + "assign device (%04x:%02x:%02x.%u) failed\n", + seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); assign_device_out: put_domain(d); @@ -846,6 +853,7 @@ long arch_do_domctl( case XEN_DOMCTL_deassign_device: { struct domain *d; + u16 seg; u8 bus, devfn; ret = -ENOSYS; @@ -864,16 +872,17 @@ long arch_do_domctl( if ( ret ) goto deassign_device_out; + seg = domctl->u.get_device_group.machine_bdf >> 24; bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff; devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff; spin_lock(&pcidevs_lock); - ret = deassign_device(d, bus, devfn); + ret = deassign_device(d, seg, bus, devfn); spin_unlock(&pcidevs_lock); if ( ret ) gdprintk(XENLOG_ERR, "XEN_DOMCTL_deassign_device: " - "deassign device (%x:%x.%x) failed\n", - bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + "deassign device (%04x:%02x:%02x.%u) failed\n", + seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); deassign_device_out: put_domain(d); --- 2011-08-25.orig/xen/drivers/passthrough/amd/pci_amd_iommu.c 2011-08-25 15:06:35.000000000 +0200 +++ 2011-08-25/xen/drivers/passthrough/amd/pci_amd_iommu.c 2011-08-25 15:06:40.000000000 +0200 @@ -288,7 +288,7 @@ static void amd_iommu_disable_domain_dev } static int reassign_device( struct domain *source, struct domain *target, - u8 bus, u8 devfn) + u16 seg, u8 bus, u8 devfn) { struct pci_dev *pdev; struct amd_iommu *iommu; @@ -296,7 +296,7 @@ static int reassign_device( struct domai struct hvm_iommu *t = domain_hvm_iommu(target); ASSERT(spin_is_locked(&pcidevs_lock)); - pdev = pci_get_pdev_by_domain(source, 0, bus, devfn); + pdev = pci_get_pdev_by_domain(source, seg, bus, devfn); if ( !pdev ) return -ENODEV; @@ -329,7 +329,7 @@ static int reassign_device( struct domai return 0; } -static int amd_iommu_assign_device(struct domain *d, u8 bus, u8 devfn) +static int amd_iommu_assign_device(struct domain *d, u16 seg, u8 bus, u8 devfn) { int bdf = (bus << 8) | devfn; int req_id = get_dma_requestor_id(bdf); @@ -344,7 +344,7 @@ static int amd_iommu_assign_device(struc ivrs_mappings[req_id].read_permission); } - return reassign_device(dom0, d, bus, devfn); + return reassign_device(dom0, d, seg, bus, devfn); } static void deallocate_next_page_table(struct page_info* pg, int level) @@ -409,9 +409,9 @@ static void amd_iommu_domain_destroy(str } static int amd_iommu_return_device( - struct domain *s, struct domain *t, u8 bus, u8 devfn) + struct domain *s, struct domain *t, u16 seg, u8 bus, u8 devfn) { - return reassign_device(s, t, bus, devfn); + return reassign_device(s, t, seg, bus, devfn); } static int amd_iommu_add_device(struct pci_dev *pdev) @@ -458,7 +458,7 @@ static int amd_iommu_remove_device(struc return 0; } -static int amd_iommu_group_id(u8 bus, u8 devfn) +static int amd_iommu_group_id(u16 seg, u8 bus, u8 devfn) { int rt; int bdf = (bus << 8) | devfn; --- 2011-08-25.orig/xen/drivers/passthrough/iommu.c 2011-08-25 15:06:35.000000000 +0200 +++ 2011-08-25/xen/drivers/passthrough/iommu.c 2011-08-25 15:06:40.000000000 +0200 @@ -165,7 +165,7 @@ int iommu_remove_device(struct pci_dev * return hd->platform_ops->remove_device(pdev); } -int assign_device(struct domain *d, u8 bus, u8 devfn) +int assign_device(struct domain *d, u16 seg, u8 bus, u8 devfn) { struct hvm_iommu *hd = domain_hvm_iommu(d); int rc = 0; @@ -174,7 +174,7 @@ int assign_device(struct domain *d, u8 b return 0; spin_lock(&pcidevs_lock); - if ( (rc = hd->platform_ops->assign_device(d, bus, devfn)) ) + if ( (rc = hd->platform_ops->assign_device(d, seg, bus, devfn)) ) goto done; if ( has_arch_pdevs(d) && !need_iommu(d) ) @@ -272,7 +272,7 @@ int iommu_unmap_page(struct domain *d, u } /* caller should hold the pcidevs_lock */ -int deassign_device(struct domain *d, u8 bus, u8 devfn) +int deassign_device(struct domain *d, u16 seg, u8 bus, u8 devfn) { struct hvm_iommu *hd = domain_hvm_iommu(d); struct pci_dev *pdev = NULL; @@ -282,7 +282,7 @@ int deassign_device(struct domain *d, u8 return -EINVAL; ASSERT(spin_is_locked(&pcidevs_lock)); - pdev = pci_get_pdev(0, bus, devfn); + pdev = pci_get_pdev(seg, bus, devfn); if ( !pdev ) return -ENODEV; @@ -293,12 +293,12 @@ int deassign_device(struct domain *d, u8 return -EINVAL; } - ret = hd->platform_ops->reassign_device(d, dom0, bus, devfn); + ret = hd->platform_ops->reassign_device(d, dom0, seg, bus, devfn); if ( ret ) { dprintk(XENLOG_ERR VTDPREFIX, - "d%d: Deassign device (%x:%x.%x) failed!\n", - d->domain_id, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + "d%d: Deassign device (%04x:%02x:%02x.%u) failed!\n", + d->domain_id, seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); return ret; } @@ -347,7 +347,7 @@ int __init iommu_setup(void) return rc; } -int iommu_get_device_group(struct domain *d, u8 bus, u8 devfn, +int iommu_get_device_group(struct domain *d, u16 seg, u8 bus, u8 devfn, XEN_GUEST_HANDLE_64(uint32) buf, int max_sdevs) { struct hvm_iommu *hd = domain_hvm_iommu(d); @@ -360,15 +360,16 @@ int iommu_get_device_group(struct domain if ( !iommu_enabled || !ops || !ops->get_device_group_id ) return 0; - group_id = ops->get_device_group_id(bus, devfn); + group_id = ops->get_device_group_id(seg, bus, devfn); spin_lock(&pcidevs_lock); for_each_pdev( d, pdev ) { - if ( (pdev->bus == bus) && (pdev->devfn == devfn) ) + if ( (pdev->seg != seg) || + ((pdev->bus == bus) && (pdev->devfn == devfn)) ) continue; - sdev_id = ops->get_device_group_id(pdev->bus, pdev->devfn); + sdev_id = ops->get_device_group_id(seg, pdev->bus, pdev->devfn); if ( (sdev_id == group_id) && (i < max_sdevs) ) { bdf = 0; --- 2011-08-25.orig/xen/drivers/passthrough/pci.c 2011-08-25 15:06:35.000000000 +0200 +++ 2011-08-25/xen/drivers/passthrough/pci.c 2011-08-25 15:06:40.000000000 +0200 @@ -441,11 +441,12 @@ void pci_release_devices(struct domain * while ( (pdev = pci_get_pdev_by_domain(d, -1, -1, -1)) ) { pci_cleanup_msi(pdev); - bus = pdev->bus; devfn = pdev->devfn; - if ( deassign_device(d, bus, devfn) ) - printk("domain %d: deassign device (%02x:%02x.%x) failed!\n", - d->domain_id, pdev->bus, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + bus = pdev->bus; + devfn = pdev->devfn; + if ( deassign_device(d, pdev->seg, bus, devfn) ) + printk("domain %d: deassign device (%04x:%02x:%02x.%u) failed!\n", + d->domain_id, pdev->seg, bus, + PCI_SLOT(devfn), PCI_FUNC(devfn)); } spin_unlock(&pcidevs_lock); } --- 2011-08-25.orig/xen/drivers/passthrough/vtd/iommu.c 2011-08-25 15:06:35.000000000 +0200 +++ 2011-08-25/xen/drivers/passthrough/vtd/iommu.c 2011-08-25 15:06:40.000000000 +0200 @@ -1573,13 +1573,13 @@ out: static int reassign_device_ownership( struct domain *source, struct domain *target, - u8 bus, u8 devfn) + u16 seg, u8 bus, u8 devfn) { struct pci_dev *pdev; int ret; ASSERT(spin_is_locked(&pcidevs_lock)); - pdev = pci_get_pdev_by_domain(source, 0, bus, devfn); + pdev = pci_get_pdev_by_domain(source, seg, bus, devfn); if (!pdev) return -ENODEV; @@ -2117,12 +2117,12 @@ int __init intel_vtd_setup(void) * If the device isn't owned by dom0, it means it already * has been assigned to other domain, or it's not exist. */ -int device_assigned(u8 bus, u8 devfn) +int device_assigned(u16 seg, u8 bus, u8 devfn) { struct pci_dev *pdev; spin_lock(&pcidevs_lock); - pdev = pci_get_pdev_by_domain(dom0, 0, bus, devfn); + pdev = pci_get_pdev_by_domain(dom0, seg, bus, devfn); if (!pdev) { spin_unlock(&pcidevs_lock); @@ -2133,7 +2133,8 @@ int device_assigned(u8 bus, u8 devfn) return 0; } -static int intel_iommu_assign_device(struct domain *d, u8 bus, u8 devfn) +static int intel_iommu_assign_device( + struct domain *d, u16 seg, u8 bus, u8 devfn) { struct acpi_rmrr_unit *rmrr; int ret = 0, i; @@ -2144,7 +2145,7 @@ static int intel_iommu_assign_device(str return -ENODEV; ASSERT(spin_is_locked(&pcidevs_lock)); - pdev = pci_get_pdev(0, bus, devfn); + pdev = pci_get_pdev(seg, bus, devfn); if (!pdev) return -ENODEV; @@ -2155,7 +2156,7 @@ static int intel_iommu_assign_device(str return -EBUSY; } - ret = reassign_device_ownership(dom0, d, bus, devfn); + ret = reassign_device_ownership(dom0, d, seg, bus, devfn); if ( ret ) goto done; @@ -2187,7 +2188,7 @@ done: return ret; } -static int intel_iommu_group_id(u8 bus, u8 devfn) +static int intel_iommu_group_id(u16 seg, u8 bus, u8 devfn) { u8 secbus; if ( find_upstream_bridge(&bus, &devfn, &secbus) < 0 ) --- 2011-08-25.orig/xen/include/xen/iommu.h 2011-08-25 15:06:23.000000000 +0200 +++ 2011-08-25/xen/include/xen/iommu.h 2011-08-25 15:06:40.000000000 +0200 @@ -74,10 +74,10 @@ int iommu_remove_device(struct pci_dev * int iommu_domain_init(struct domain *d); void iommu_dom0_init(struct domain *d); void iommu_domain_destroy(struct domain *d); -int device_assigned(u8 bus, u8 devfn); -int assign_device(struct domain *d, u8 bus, u8 devfn); -int deassign_device(struct domain *d, u8 bus, u8 devfn); -int iommu_get_device_group(struct domain *d, u8 bus, u8 devfn, +int device_assigned(u16 seg, u8 bus, u8 devfn); +int assign_device(struct domain *d, u16 seg, u8 bus, u8 devfn); +int deassign_device(struct domain *d, u16 seg, u8 bus, u8 devfn); +int iommu_get_device_group(struct domain *d, u16 seg, u8 bus, u8 devfn, XEN_GUEST_HANDLE_64(uint32) buf, int max_sdevs); /* iommu_map_page() takes flags to direct the mapping operation. */ @@ -125,14 +125,14 @@ struct iommu_ops { void (*dom0_init)(struct domain *d); int (*add_device)(struct pci_dev *pdev); int (*remove_device)(struct pci_dev *pdev); - int (*assign_device)(struct domain *d, u8 bus, u8 devfn); + int (*assign_device)(struct domain *d, u16 seg, u8 bus, u8 devfn); void (*teardown)(struct domain *d); int (*map_page)(struct domain *d, unsigned long gfn, unsigned long mfn, unsigned int flags); int (*unmap_page)(struct domain *d, unsigned long gfn); int (*reassign_device)(struct domain *s, struct domain *t, - u8 bus, u8 devfn); - int (*get_device_group_id)(u8 bus, u8 devfn); + u16 seg, u8 bus, u8 devfn); + int (*get_device_group_id)(u16 seg, u8 bus, u8 devfn); void (*update_ire_from_apic)(unsigned int apic, unsigned int reg, unsigned int value); void (*update_ire_from_msi)(struct msi_desc *msi_desc, struct msi_msg *msg); void (*read_msi_from_ire)(struct msi_desc *msi_desc, struct msi_msg *msg);