[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v17 4/8] remus drbd: Implement remus drbd replicated disk
Implement remus-drbd-replicated-checkpointing-disk based on generic remus devices framework. Signed-off-by: Lai Jiangshan <laijs@xxxxxxxxxxxxxx> Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx> Signed-off-by: Yang Hongyang <yanghy@xxxxxxxxxxxxxx> --- tools/hotplug/Linux/Makefile | 1 + tools/hotplug/Linux/block-drbd-probe | 85 ++++++++++++ tools/libxl/Makefile | 2 +- tools/libxl/libxl.c | 2 + tools/libxl/libxl_internal.h | 3 + tools/libxl/libxl_remus_disk_drbd.c | 243 +++++++++++++++++++++++++++++++++++ 6 files changed, 335 insertions(+), 1 deletion(-) create mode 100755 tools/hotplug/Linux/block-drbd-probe create mode 100644 tools/libxl/libxl_remus_disk_drbd.c diff --git a/tools/hotplug/Linux/Makefile b/tools/hotplug/Linux/Makefile index 721f8c0..15d1b37 100644 --- a/tools/hotplug/Linux/Makefile +++ b/tools/hotplug/Linux/Makefile @@ -24,6 +24,7 @@ XEN_SCRIPTS += xen-hotplug-cleanup XEN_SCRIPTS += external-device-migrate XEN_SCRIPTS += vscsi XEN_SCRIPTS += block-iscsi +XEN_SCRIPTS += block-drbd-probe XEN_SCRIPTS += $(XEN_SCRIPTS-y) XEN_SCRIPT_DATA = xen-script-common.sh locking.sh logging.sh diff --git a/tools/hotplug/Linux/block-drbd-probe b/tools/hotplug/Linux/block-drbd-probe new file mode 100755 index 0000000..3a3d446 --- /dev/null +++ b/tools/hotplug/Linux/block-drbd-probe @@ -0,0 +1,85 @@ +#! /bin/bash +# +# Copyright (C) 2014 FUJITSU LIMITED +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of version 2.1 of the GNU Lesser General Public +# License as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Usage: +# block-drbd-probe devicename +# +# Return value: +# 0: the device is drbd device +# 1: the device is not drbd device +# 2: unkown error +# 3: the drbd device does not use protocol D +# 4: the drbd device is not ready + +drbd_res= + +function get_res_name() +{ + local drbd_dev=$1 + local drbd_dev_list=($(drbdadm sh-dev all)) + local drbd_res_list=($(drbdadm sh-resource all)) + local temp_drbd_dev temp_drbd_res + local found=0 + + for temp_drbd_dev in ${drbd_dev_list[@]}; do + if [[ "$temp_drbd_dev" == "$drbd_dev" ]]; then + found=1 + break + fi + done + + if [[ $found -eq 0 ]]; then + return 1 + fi + + for temp_drbd_res in ${drbd_res_list[@]}; do + temp_drbd_dev=$(drbdadm sh-dev $temp_drbd_res) + if [[ "$temp_drbd_dev" == "$drbd_dev" ]]; then + drbd_res="$temp_drbd_res" + return 0 + fi + done + + # OOPS + return 2 +} + +get_res_name $1 +rc=$? +if [[ $rc -ne 0 ]]; then + exit $rc +fi + +# check protocol +drbdsetup $1 show | grep -q "protocol D;" +if [[ $? -ne 0 ]]; then + exit 3 +fi + +# check connect status +state=$(drbdadm cstate "$drbd_res") +if [[ "$state" != "Connected" ]]; then + exit 4 +fi + +# check role +role=$(drbdadm role "$drbd_res") +if [[ "$role" != "Primary/Secondary" ]]; then + exit 4 +fi + +exit 0 diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 202f1bb..ba10ab7 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -56,7 +56,7 @@ else LIBXL_OBJS-y += libxl_nonetbuffer.o endif -LIBXL_OBJS-y += libxl_remus_device.o +LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index cd936c2..78a4752 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -790,6 +790,7 @@ static void remus_failover_cb(libxl__egc *egc, static const libxl__remus_device_subkind_ops *remus_ops[] = { &remus_device_nic, + &remus_device_drbd_disk, NULL, }; @@ -831,6 +832,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, } rds->device_kind_flags |= LIBXL__REMUS_DEVICE_NIC; } + rds->device_kind_flags |= LIBXL__REMUS_DEVICE_DISK; rds->ao = ao; rds->egc = egc; diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 5f2ee05..980f300 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -140,6 +140,7 @@ typedef struct libxl__aop_occurred libxl__aop_occurred; typedef struct libxl__osevent_hook_nexus libxl__osevent_hook_nexus; typedef struct libxl__osevent_hook_nexi libxl__osevent_hook_nexi; typedef struct libxl__remus_netbuf_state libxl__remus_netbuf_state; +typedef struct libxl__remus_drbd_state libxl__remus_drbd_state; _hidden void libxl__alloc_failed(libxl_ctx *, const char *func, size_t nmemb, size_t size) __attribute__((noreturn)); @@ -377,6 +378,7 @@ struct libxl__ctx { libxl_version_info version_info; libxl__remus_netbuf_state *rns; + libxl__remus_drbd_state *drbd_state; }; typedef struct { @@ -2664,6 +2666,7 @@ _hidden void libxl__remus_devices_commit(libxl__egc *egc, libxl__remus_device_state *rds); extern const libxl__remus_device_subkind_ops remus_device_nic; +extern const libxl__remus_device_subkind_ops remus_device_drbd_disk; _hidden int libxl__netbuffer_enabled(libxl__gc *gc); diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c new file mode 100644 index 0000000..88a1984 --- /dev/null +++ b/tools/libxl/libxl_remus_disk_drbd.c @@ -0,0 +1,243 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author Lai Jiangshan <laijs@xxxxxxxxxxxxxx> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +/*** drbd implementation ***/ +const int DRBD_SEND_CHECKPOINT = 20; +const int DRBD_WAIT_CHECKPOINT_ACK = 30; + +struct libxl__remus_drbd_state { + libxl__ao *ao; + char *drbd_probe_script; +}; + +typedef struct libxl__remus_drbd_disk { + int ctl_fd; + int ackwait; + const char *path; + + libxl__async_exec_state aes; + libxl__ev_child child; +} libxl__remus_drbd_disk; + +/*----- helper functions, for async calls -----*/ +static void drbd_async_call(libxl__remus_device *dev, + void func(libxl__remus_device *), + libxl__ev_child_callback callback) +{ + int pid = -1; + STATE_AO_GC(dev->rds->ao); + + /* Fork and call */ + pid = libxl__ev_child_fork(gc, &dev->child, callback); + if (pid == -1) { + LOG(ERROR, "unable to fork"); + goto out; + } + + if (!pid) { + /* child */ + func(dev); + /* notreached */ + abort(); + } + + return; + +out: + dev->callback(dev->rds->egc, dev, ERROR_FAIL); +} + +/*----- init() and cleanup() -----*/ +static int drbd_init(libxl__remus_device_state *rds) +{ + libxl__remus_drbd_state *drbd_state; + + STATE_AO_GC(rds->ao); + + GCNEW(drbd_state); + CTX->drbd_state = drbd_state; + drbd_state->ao = ao; + drbd_state->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe", + libxl__xen_script_dir_path()); + + return 0; +} + +static void drbd_cleanup(libxl__remus_device_state *rds) +{ + return; +} + +/*----- match(), setup() and teardown() -----*/ + +/* callbacks */ +static void match_async_exec_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status); + +/* implementations */ + +static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev) +{ + int arraysize, nr = 0; + const libxl_device_disk *disk = dev->backend_dev; + libxl__async_exec_state *aes = &dev->aes; + STATE_AO_GC(dev->rds->ao); + + libxl__remus_drbd_state *drbd_state = CTX->drbd_state; + /* setup env & args */ + arraysize = 1; + GCNEW_ARRAY(aes->env, arraysize); + aes->env[nr++] = NULL; + assert(nr <= arraysize); + + arraysize = 3; + nr = 0; + GCNEW_ARRAY(aes->args, arraysize); + aes->args[nr++] = drbd_state->drbd_probe_script; + aes->args[nr++] = disk->pdev_path; + aes->args[nr++] = NULL; + assert(nr <= arraysize); + + aes->ao = drbd_state->ao; + aes->what = GCSPRINTF("%s %s", aes->args[0], aes->args[1]); + aes->timeout_ms = LIBXL_HOTPLUG_TIMEOUT * 1000; + aes->callback = match_async_exec_cb; + aes->stdfds[0] = -1; + aes->stdfds[1] = -1; + aes->stdfds[2] = -1; + + if (libxl__async_exec_start(gc, aes)) + goto out; + + return; + +out: + dev->callback(egc, dev, ERROR_FAIL); +} + +static void drbd_match(libxl__remus_device *dev) +{ + match_async_exec(dev->rds->egc, dev); +} + +static void match_async_exec_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status) +{ + libxl__remus_device *dev = CONTAINER_OF(aes, *dev, aes); + + if (status) { + dev->callback(egc, dev, ERROR_REMUS_DEVOPS_NOT_MATCH); + } else { + dev->callback(egc, dev, 0); + } +} + +static void drbd_setup(libxl__remus_device *dev) +{ + libxl__remus_drbd_disk *drbd_disk; + const libxl_device_disk *disk = dev->backend_dev; + STATE_AO_GC(dev->rds->ao); + + GCNEW(drbd_disk); + dev->concrete_data = drbd_disk; + drbd_disk->path = disk->pdev_path; + drbd_disk->ackwait = 0; + drbd_disk->ctl_fd = open(drbd_disk->path, O_RDONLY); + if (drbd_disk->ctl_fd < 0) + dev->callback(dev->rds->egc, dev, ERROR_FAIL); + else + dev->callback(dev->rds->egc, dev, 0); +} + +static void drbd_teardown(libxl__remus_device *dev) +{ + libxl__remus_drbd_disk *drbd_disk = dev->concrete_data; + + close(drbd_disk->ctl_fd); + dev->callback(dev->rds->egc, dev, 0); +} + +/*----- checkpointing APIs -----*/ + +/* callbacks */ +static void chekpoint_async_call_done(libxl__egc *egc, + libxl__ev_child *child, + pid_t pid, int status); + +/* API implementations */ + +/* this op will not wait and block, so implement as sync op */ +static void drbd_postsuspend(libxl__remus_device *dev) +{ + libxl__remus_drbd_disk *rdd = dev->concrete_data; + + if (!rdd->ackwait) { + if (ioctl(rdd->ctl_fd, DRBD_SEND_CHECKPOINT, 0) <= 0) + rdd->ackwait = 1; + } + + dev->callback(dev->rds->egc, dev, 0); +} + +static void drbd_preresume_async(libxl__remus_device *dev) +{ + libxl__remus_drbd_disk *rdd = dev->concrete_data; + int ackwait = rdd->ackwait; + + if (ackwait) { + ioctl(rdd->ctl_fd, DRBD_WAIT_CHECKPOINT_ACK, 0); + ackwait = 0; + } + + _exit(ackwait); +} + +static void drbd_preresume(libxl__remus_device *dev) +{ + drbd_async_call(dev, drbd_preresume_async, chekpoint_async_call_done); +} + +static void chekpoint_async_call_done(libxl__egc *egc, + libxl__ev_child *child, + pid_t pid, int status) +{ + libxl__remus_device *dev = CONTAINER_OF(child, *dev, child); + libxl__remus_drbd_disk *rdd = dev->concrete_data; + STATE_AO_GC(dev->rds->ao); + + if (WIFEXITED(status)) { + rdd->ackwait = WEXITSTATUS(status); + dev->callback(egc, dev, 0); + } else { + dev->callback(egc, dev, ERROR_FAIL); + } +} + +const libxl__remus_device_subkind_ops remus_device_drbd_disk = { + .kind = LIBXL__REMUS_DEVICE_DISK, + .init = drbd_init, + .cleanup = drbd_cleanup, + .match = drbd_match, + .setup = drbd_setup, + .teardown = drbd_teardown, + .postsuspend = drbd_postsuspend, + .preresume = drbd_preresume, +}; -- 1.9.1 _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxx http://lists.xen.org/xen-devel
|
Lists.xenproject.org is hosted with RackSpace, monitoring our |