VGA card:
root@cknowsvr01:/home/diederik# lspci | grep -i vga
02:00.0 VGA compatible controller: NVIDIA Corporation NV44 [GeForce 6200 TurboCache] (rev a1)


Uninstalled the nvidia proprietary driver (Debian package) and installed nouveau and rebooted my server, which resulted in the following kernel modules being loaded:
diederik@cknowsvr01:~$ lsmod | grep nou
nouveau              1544192  1
mxm_wmi                16384  1 nouveau
video                  40960  1 nouveau
ttm                    98304  1 nouveau
drm_kms_helper        155648  1 nouveau
drm                   360448  4 nouveau,ttm,drm_kms_helper
wmi                    16384  2 mxm_wmi,nouveau
button                 16384  1 nouveau
i2c_algo_bit           16384  2 igb,nouveau

After reboot I did: 
root@cknowsvr01:/home/diederik# xl pci-assignable-list
libxl: error: libxl_pci.c:409:libxl_device_pci_assignable_list: Looks like pciback driver not loaded

So forgot to load the xen-pciback driver, so fixed that:
root@cknowsvr01:/home/diederik# modprobe xen-pciback

root@cknowsvr01:/home/diederik# xl pci-assignable-list

Then tried to create a vm in which I had enabled pci-passthrough:
root@cknowsvr01:/home/diederik# xl create -c /etc/xen/tradestation.home.cknow.org.cfg 
Parsing config from /etc/xen/tradestation.home.cknow.org.cfg
libxl: notice: libxl_numa.c:518:libxl__get_numa_candidate: NUMA placement failed, performance might be affected
libxl: error: libxl_qmp.c:287:qmp_handle_error_response: received an error message from QMP server: Could not set password
libxl: error: libxl_pci.c:1225:libxl__device_pci_add: PCI device 0:2:0.0 is not assignable
libxl: error: libxl_pci.c:1304:libxl__add_pcidevs: libxl_device_pci_add failed: -3
libxl: error: libxl_create.c:1461:domcreate_attach_devices: unable to add pci devices
libxl: error: libxl.c:1575:libxl__destroy_domid: non-existant domain 1
libxl: error: libxl.c:1534:domain_destroy_callback: unable to destroy guest with domid 1
libxl: error: libxl.c:1463:domain_destroy_cb: destruction of domain 1 failed
root@cknowsvr01:/home/diederik# xl list
Name                                        ID   Mem VCPUs      State   Time(s)
Domain-0                                     0 112773    32     r-----      54.3

which failed.
Realizing I forgot to add my vga card, so tried to fix that:
root@cknowsvr01:/home/diederik# xl pci-assignable-add 02:00.0
at which my system hang and eventually my ssh-session got disconnected and am unable to connect again. Plugging in a physical keyboard didn't help and a hard reset seems to be the only way out.