Re: [Xen-users] pv guest fails to start after 4.2 upgrade

[Travis Osterman <tosterman@xxxxxxxxx>]

> So I asked around and there is basically no way the qemu-xen-trad blktap
> module should be active for any reason in a Xen 4.2 system with xl. That
> is the deadest of dead code...
>
> Could you send a full set of logs (xl -vvv create, /var/log/xen/*) and
> your guest config for this new issue please.

Here is the full report from a freshly rebooted server.

/*
 * Fresh reboot, all xl-related services off
 */

# emerge --info
http://pastebin.com/XBK7fwXm

# lsof -X | grep -i tap
(nothing)

# ls -R /var/run/{xen,tap}
/var/run/tap:

/var/run/xen:

# ps -e | grep "xl\|qemu\|xen"
   48 ?        00:00:00 xenwatch
   49 ?        00:00:00 xenbus
  713 ?        00:00:00 xen_pciback_wor

/*
 * bringing up xenstored
 */
# /etc/init.d/xenstored start
 * Starting xenstored daemon ...
 * Setting domain0 name record

# lsof -X | grep -i tap
(nothing)

# ls -R /var/run/{xen,tap}
/var/run/tap:
st
/var/run/xen:

# ps -e | grep "xl\|qemu\|xen"
   48 ?        00:00:00 xenwatch
   49 ?        00:00:00 xenbus
  713 ?        00:00:00 xen_pciback_wor
 9264 ?        00:00:00 xenstored

/*
 * bringing up xenconsoled (previously in default runlevel)
 */
# /etc/init.d/xenconsoled start
 * Starting xenconsoled daemon ...
                       [ ok ]
# lsof -X | grep -i tap
# ls -R /var/run/{xen,tap}
/var/run/tap:

/var/run/xen:
xen ~ # ps -e | grep "xl\|qemu\|xen"
   48 ?        00:00:00 xenwatch
   49 ?        00:00:00 xenbus
  713 ?        00:00:00 xen_pciback_wor
 9264 ?        00:00:00 xenstored
 9288 ?        00:00:00 xenconsoled

/*
 * just dom0 started
 */
# xl list
Name                                        ID   Mem VCPUs      State   Time(s)
Domain-0                                     0 32106     8     r-----      52.7

# xl info
http://pastebin.com/fx0F4cbm

# xl dmesg
http://pastebin.com/cX9b3Yt5

# grep ^[^[:space:]*#] /data/xen/prime.pv
name = "prime"
memory = 2048
vcpus=8
vif=['bridge=xenbr0,mac=00:16:3E:1F:B8:0E,ip=192.168.1.1']
disk = ['file:/data/xen/prime.img,xvda,w']
kernel = "/usr/lib/xen/boot/pv-grub-x86_64.gz"
extra = "(hd0,0)/grub/menu.lst xencons=tty iommu=soft"
on_poweroff = 'destroy'
on_reboot   = 'restart'
on_crash    = 'restart'

/*
 * PV using pvgrub (with manually creating /var/run/xen
 */
# xl -vvv create /data/xen/prime.pv
http://pastebin.com/FMTWAqJd

# xl list
Name                                        ID   Mem VCPUs      State   Time(s)
Domain-0                                     0 29606     8     r-----      56.4
prime                                        1  2048     8     -b----      33.9

/*
 * I needed to attach a couple NICs to finish brining my router online
 */
# xl pci-attach prime 0000:01:00.0
# xl pci-attach prime 0000:0a:00.0


/*
 * libblktapctl is up now
 */
# lsof -X | grep -i tap
xl        9387      root  mem       REG              253,0      30400
  1462266 /usr/lib64/libblktapctl.so.1.0.0
xl        9387 9388 root  mem       REG              253,0      30400
  1462266 /usr/lib64/libblktapctl.so.1.0.0


# ls -R /var/run/{xen,tap}
/var/run/tap:

/var/run/xen:
qmp-libxl-1

# ps -e | grep "xl\|qemu\|xen"
   48 ?        00:00:00 xenwatch
   49 ?        00:00:00 xenbus
  713 ?        00:00:00 xen_pciback_wor
 9264 ?        00:00:00 xenstored
 9288 ?        00:00:00 xenconsoled
 9327 ?        00:00:00 qemu-system-i38
 9387 ?        00:00:00 xl


 # ls -1 /var/log/xen/*prime*.log
/var/log/xen/qemu-dm-prime.log
/var/log/xen/xl-prime.log

# cat /var/log/xen/qemu-dm-prime.log
xen ~ # cat /var/log/xen/xl-prime.log
Waiting for domain prime (domid 1) to die [pid 9387]
libxl: debug: libxl_event.c:512:libxl__ev_xswatch_register: watch
w=0x627c10 wpath=@releaseDomain token=3/0: register slotnum=3
libxl: debug: libxl_event.c:457:watchfd_callback: watch w=0x627c10
wpath=@releaseDomain token=3/0: event epath=@releaseDomain
libxl: debug: libxl.c:1015:domain_death_xswatch_callback:
[evg=0x6299d0:1] from domid=1 nentries=1 rc=1
libxl: debug: libxl.c:1026:domain_death_xswatch_callback:
[evg=0x6299d0:1]   got=domaininfos[0] got->domain=1
libxl: debug: libxl.c:1053:domain_death_xswatch_callback:  exists
shutdown_reported=0 dominf.flags=ffff0020
libxl: debug: libxl.c:1019:domain_death_xswatch_callback: [evg=0] all reported
libxl: debug: libxl.c:1083:domain_death_xswatch_callback: domain death
search done


/*************************************************************************
 * PV with pygrub
 */

# grep ^[^[:space:]*#] /tank/xen/config/spot.pv.orig
name = "spot"
memory = 2048
bootloader = "pygrub"
vcpus=4
vif=['bridge=xenbr0,mac=00:16:3E:21:56:66,ip=192.168.1.30']
disk = ['file:/tank/xen/dev/spot.img,xvda,w']

# which pygrub
/usr/bin/pygrub

/*
 * boot via pygrub using img file
 */
# xl -vvv create /tank/xen/config/spot.pv.orig
http://pastebin.com/x1GHPXTP

/*
 * in the console
 */
http://pastebin.com/kiGchPx7
Key type dns_resolver registered
registered taskstats version 1
XENBUS: Waiting for devices to initialise:
25s...20s...15s...10s...5s...0s...235s...230s...225s...220s...215s...210s...
............
   :: Scanning for raid10...raid10 loaded.
   :: Scanning for linear...linear loaded.
   :: Scanning for multipath...multipath loaded.
   :: Scanning for e1000...e1000 loaded.
   :: Scanning for tg3...libphy, tg3 loaded.
   :: Scanning for scsi_transport_iscsi...scsi_transport_iscsi loaded.
   :: Scanning for libiscsi...libiscsi loaded.
   :: Scanning for cbc...cbc loaded.
   :: Scanning for aes-x86_64...aes-x86_64 loaded.
   :: Scanning for xts...gf128mul, xts loaded.
>> Hint: Use parameter scandelay[=seconds] if you need waiting here
>> Determining root device...
!! Block device /dev/xvda2 is not a valid root device...
!! Could not find the root block device in .
   Please specify another value or: press Enter for the same, type
"shell" for a shell, or "q" to skip...
root block device() :: shell
# ls /dev/sd*
ls: /dev/sd*: No such file or directory
# ls /dev/hd*
ls: /dev/hd*: No such file or directory
# ls /dev/xv*
ls: /dev/xv*: No such file or directory
# exit

/*
 * Logs
 */

 # ls -1 /var/log/xen/*spot*.log
/var/log/xen/qemu-dm-spot.log
/var/log/xen/xl-spot.log

 # cat /var/log/xen/qemu-dm-spot.log
(empty)

 # cat /var/log/xen/xl-spot.log
Waiting for domain spot (domid 2) to die [pid 9749]
libxl: debug: libxl_event.c:512:libxl__ev_xswatch_register: watch
w=0x628a30 wpath=@releaseDomain token=3/0: register slotnum=3
libxl: debug: libxl_event.c:457:watchfd_callback: watch w=0x628a30
wpath=@releaseDomain token=3/0: event epath=@releaseDomain
libxl: debug: libxl.c:1015:domain_death_xswatch_callback:
[evg=0x625020:2] from domid=2 nentries=1 rc=1
libxl: debug: libxl.c:1026:domain_death_xswatch_callback:
[evg=0x625020:2]   got=domaininfos[0] got->domain=2
libxl: debug: libxl.c:1053:domain_death_xswatch_callback:  exists
shutdown_reported=0 dominf.flags=ffff0020
libxl: debug: libxl.c:1019:domain_death_xswatch_callback: [evg=0] all reported
libxl: debug: libxl.c:1083:domain_death_xswatch_callback: domain death
search done

/*
 * loop workaround
 */

# losetup /dev/loop0 /tank/xen/dev/spot.img
# xl -vvv create /tank/xen/config/spot.pv
http://pastebin.com/nDmmftqA

(works fine)

I hope this helps.  Sorry for the long email.

-- T

_______________________________________________
Xen-users mailing list
Xen-users@xxxxxxxxxxxxx
http://lists.xen.org/xen-users