[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [OSSTEST PATCH 09/11] mg-schema-test-database: New script



This allows a user in non-standalone mode to make a whole new test
database, which is largely a clone of the original database.

The new db refers to the same resources (hosts), and more-or-less
safely borrows some of those hosts.

Currently we don't do anything about the queue and owner daemons.
This means that queue-daemon-based resource allocation is broken when
clients are pointed at the test db.  But non-queue-based allocation
(eg, ./mg-allocate without -U) works, and the test db can be used for
db-related experiments and even support individual ts-* scripts (other
than ts-hosts-allocate of course).

Signed-off-by: Ian Jackson <Ian.Jackson@xxxxxxxxxxxxx>
---
 mg-schema-test-database |  458 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 458 insertions(+)
 create mode 100755 mg-schema-test-database

diff --git a/mg-schema-test-database b/mg-schema-test-database
new file mode 100755
index 0000000..64acdcf
--- /dev/null
+++ b/mg-schema-test-database
@@ -0,0 +1,458 @@
+#!/bin/bash
+#
+# usages:
+#
+#
+#  ./mg-schema-test-database create [_SUFFIX] [TASK...] \
+#              [-fMINFLIGHT | -f-NUMFLIGHTS]
+#
+# does `drop' and then creates
+#   - the database    osstestdb_test_SUFFIX
+#   - a file          local-config.test-database_SUFFIX
+#
+# default for SUFFIX is your local username
+#
+# Resources owned in the main db by a task in the list of specified
+# TASKs become idle in the test copy.  Others become allocated to
+# a specially-created `owned by someone in real db' task.
+#
+#
+#  ./mg-schema-test-database drop [_SUFFIX]
+#
+# deletes your test database and removes the local-config file
+#
+#
+
+set -e -o posix ${OSSTEST_DEBUG:+-x}
+
+. ./cri-getconfig
+. ./mgi-common
+
+if [ $# -lt 1 ]; then fail "need operation"; fi
+
+cmd="$1"; shift
+
+localconfig=local-config.test-database
+
+maindbname=$(perl -we '
+       use Osstest;
+       use Osstest::Executive;
+       use DBI;
+       csreadconfig();
+       print $dbh_tests->{pg_db},"\n"
+            or die $!;
+')
+
+parse_only_suffix () {
+       for arg in "$@"; do
+               case "$arg" in
+               _*)     suffix="$arg" ;;
+               *)      fail 'bad usage' ;;
+               esac
+       done
+}
+
+dbname () {
+       dbname="${maindbname}_test$suffix"
+       t="tmp/testdb$suffix"
+       tcfg=local-config.test-database$suffix
+}
+
+psql_query_internal () {
+       $(get_psql_cmd) -At -R' ' -f- "$@"
+}
+
+psql_query () {
+        if [ x$OSSTEST_DEBUG != x ]; then
+               tee /dev/stderr | psql_query_internal "$@"
+       else
+               psql_query_internal "$@"
+       fi
+}
+
+psql_do_cmd () {
+       echo "$(get_psql_cmd) ${OSSTEST_DEBUG:+-e -a}" 
+}
+
+psql_do () {
+       $(psql_do_cmd) -q -f- "$@"
+}
+
+moretasks () {
+       local ifnone="$1"; shift
+       local where="$1"; shift
+       local r
+       r=$(
+               psql_query "$@" <<END
+                       SELECT taskid
+                         FROM tasks
+                        $where
+END
+       )
+       if [ "x$r" = x ]; then
+               local m="no tasks matched \`$arg'"
+               case $ifnone in
+               error)
+                       fail "error: $m"
+                       ;;
+               warning)
+                       echo >&2 "warning: $m"
+                       ;;
+               *)
+                       fail-bad-moretasks-ifnone-"$ifnone"
+                       ;;
+               esac
+       fi
+
+       tasks+=" $r"
+}
+
+withtest () {
+       OSSTEST_CONFIG="$test_cfg_setting" "$@"
+}
+
+each_copy_table () {
+       local tab=$1; shift
+       local cond=$1; shift
+
+       p=$t.tabledata.$tab
+       rm -f $p
+
+       cat <<END >>$t.export
+               \\COPY (SELECT * FROM $tab WHERE $cond) TO $p $copyhow
+END
+       cat <<END >>$t.import
+               \\COPY $tab FROM $p $copyhow
+END
+}
+
+make_xdbref_task () {
+       local refkey=$1; shift
+       local comment=$1; shift
+       local refinfo=$1; shift
+       echo "
+               INSERT INTO tasks
+                       (type, refkey, username, comment, live, refinfo)
+                 VALUES ('xdbref','$refkey','$username@$nodename',
+                         '$comment','t','$refinfo');
+       "
+}
+
+taskid () {
+       local type=$1; shift
+       local refkey=$1; shift
+       local xcond=$1
+       echo "(SELECT taskid FROM tasks WHERE
+               type='$type' AND refkey='$refkey' $xcond)"
+}
+borrowtaskid () {
+       local bt=$1
+       taskid xdbref $dbname "
+               AND live AND refinfo IS NOT NULL AND refinfo='$bt'
+       "
+}
+
+username=`whoami`
+nodename=`uname -n`
+suffix=_$username
+invocation_now=`date +%s`
+
+case "$cmd" in
+
+#========== CREATE ==========
+
+create)
+       #---------- argument parsing ----------
+
+       tasks=''
+       minflight=-1000
+       for arg in "$@"; do
+               case "$arg" in
+               *@*)
+                       moretasks warning                       \
+                               "WHERE type = 'static'
+                                  AND refkey LIKE :'pattern'"  \
+                               -v pattern="${arg//\*/%}"       \
+                       ;;
+               *" "*" "*)
+                       local rhs="${arg#* }"
+                       moretasks error                         \
+                               "WHERE taskid = :'taskid'
+                                  AND type = :'type'
+                                  AND refkey = :'refkey'"      \
+                               -v taskid="${arg%% *}"          \
+                               -v type="${rhs%% *}"            \
+                               -v refkey="${rhs#* }"
+                       ;;
+               _)      suffix="$arg"
+                       ;;
+               -f*)    minflight="${arg#-f}"
+                       ;;
+               *)      fail "bad arg to create"
+                       ;;
+               esac
+       done
+
+       if [ "x$tasks" = x ]; then
+               moretasks error                                 \
+                       "WHERE type = 'static'
+                          AND refkey = :'refkey'"              \
+                       -v refkey="$(whoami)@$(uname -n)"
+       fi
+
+       tasks_cond=${tasks// / OR T=}
+       tasks_cond=${tasks_cond# OR }
+
+       case "$minflight" in
+       -*)
+               minflight=$( psql_query <<END
+                        SELECT flight FROM
+                                (SELECT flight FROM flights
+                                        ORDER BY flight DESC
+                                        LIMIT ${minflight#-})
+                               AS last
+                                ORDER BY FLIGHT ASC
+                                LIMIT 1
+END
+               )
+               ;;
+       esac
+
+       #---------- preparation and data-gathering ----------
+
+       bad=$(  psql_query <<END
+               SELECT * FROM tasks WHERE (${tasks_cond//T/taskid})
+                               AND NOT live
+END
+       )
+       case "$bad" in
+       *[0-9]*)
+               fail "Borrowing from NON-LIVE TASKS $bad"
+               ;;
+       esac
+
+       # drop any previous test db
+       "$0" drop $suffix
+
+       dbname
+
+       printf "Setting up %s (minflight=%d, tasks=%s)...\n" \
+               $dbname "$minflight" "${tasks# }"
+
+       # create the config overlay
+       perl >$tcfg.tmp -we '
+               use Osstest;
+               use Osstest::Executive;
+               use DBI;
+               csreadconfig();
+               print "ExecutiveDbname_osstestdb ".
+                       "dbname='$dbname';".
+                       "host=$dbh_tests->{pg_host};".
+                       "user=$dbh_tests->{pg_user};".
+                       "port=$dbh_tests->{pg_port}\n"
+                       or die $!;
+       '
+       cat >>$tcfg.tmp <<END
+OwnerDaemonHost $ctrlhost
+QueueDaemonHost $ctrlhost
+OwnerDaemonPort ${ctrlports%,*}
+QueueDaemonPort ${ctrlports#*,}
+END
+       mv -f $tcfg.tmp $tcfg
+
+       # this makes `withtest' work
+       test_cfg_setting="$(perl -we '
+                       use Osstest;
+                       print globalconfigfiles() or die $!;
+               '):$tcfg"
+
+       # Extract the schema for reference
+       $(get_pgdump_cmd) -s -O -x >$t.schema
+
+       # Keep a copy as it came from dump, for comparison
+       cp $t.schema $t.schema.orig
+
+       # http://www.postgresql.org/message-id/26790.1306355327@xxxxxxxxxxxxx
+       perl -i~ -pe '
+               s/^/--/ if
+                       m/^CREATE EXTENSION IF NOT EXISTS plpgsql / ||
+                       m/^COMMENT ON EXTENSION plpgsql /;
+       ' $t.schema
+
+       printf "Tables:"
+
+       # What tables are there ?
+       perl -ne <$t.schema >$t.tablevars '
+               if (m/^CREATE SEQUENCE (\w+)/) {
+                       print "sequences+=\" $1\"\n";
+               } elsif (m/^CREATE TABLE (\w+)/) {
+                       $table=$1;
+               } elsif (m/^\s*flight\s+integer/) {
+                       print "ftables+=\" $table\"\n";
+               } elsif ($table && m/^\)\;$/) {
+                       print "tables+=\" $table\"\n";
+               }
+       '
+
+       . $t.tablevars
+
+       >$t.tablesortlist
+
+       for table in $tables; do
+               LC_MESSAGES=C $(get_psql_cmd) <<END >$t.display.$table
+                       \d $table
+END
+               echo >>$t.tablesortlist "$table $table"
+               perl -ne <$t.display.$table >>$t.tablesortlist '
+                       next unless m/^Foreign-key constraints:/ ... m/^\S/;
+                       next if m/DEFERRABLE/;
+                       next unless m/FOREIGN KEY.*REFERENCES (\w+)/;
+                       print "$1 '"$table"'\n" or die $!;
+               '
+               printf " $table"
+       done
+
+       tables=$(tsort <$t.tablesortlist)
+
+       # We don't want to set the permissions
+       perl <executive-postgresql-schema >$t.new-schema -pe '
+               s/^/--/ if
+                       m/^ALTER TABLE .* OWNER TO / ||
+                       m/^GRANT |^REVOKE /
+       '
+
+       #---------- create test db ----------
+
+       psql_do <<END
+               CREATE DATABASE $dbname;
+END
+       $(withtest get_psql_cmd) -q -f $t.new-schema
+
+       printf ".\n"
+
+       # Schema should now be identical to main DB
+       $(withtest get_pgdump_cmd) -s -O -x >$t.schema.created
+       diff -u $t.schema.orig $t.schema.created
+
+       #---------- mark resources that we are going to borrow ----------
+
+       for task in $tasks; do
+               psql_do <<END
+                       BEGIN;
+                       $(make_xdbref_task $dbname 'borrowed for test db' $task)
+                       UPDATE resources SET owntaskid = $(borrowtaskid $task)
+                               WHERE owntaskid=$task;
+                       COMMIT;
+END
+       done
+
+       #---------- copy data from live to test db ----------
+
+       copyhow="CSV HEADER NULL e'\\\\n'"
+
+       rm -f $t.import $t.export
+
+       cat >>$t.import <<END
+               \o $t.import-output
+               BEGIN;
+               SET CONSTRAINTS ALL DEFERRED;
+END
+
+       $(get_pgdump_cmd) -a -O -x ${sequences// / -t } >$t.sequences-import
+       perl <$t.sequences-import >>$t.import -ne '
+               next if m/^--/;
+               next if m/^SET /;
+               next unless m/\S/;
+               print or die $!;
+       '
+
+       for table in $tables; do
+               case " $ftables " in
+               *" $table "*)   condition="flight >= $minflight" ;;
+               *)              condition="1=1" ;;
+               esac
+               each_copy_table $table "$condition"
+       done
+
+       # As we copy, we note everything we're not borrowing as
+       # belonging to the parent db.
+       cat >>$t.import <<END
+               $(make_xdbref_task $maindbname 'not borrowed' '')
+               UPDATE resources
+                       SET owntaskid = $(taskid xdbref $maindbname)
+                       WHERE owntaskid != $(borrowtaskid $task);
+               COMMIT;
+END
+
+       printf "Copy..."
+
+       printf "export..."
+       $(psql_do_cmd) -f $t.export
+
+       printf "import..."
+       $(withtest psql_do_cmd) -f $t.import
+
+       rm -f $t.tabledata.*
+
+       #---------- actually borrow resources ----------
+
+       printf "borrow..."
+
+       for task in $tasks; do
+               withtest psql_do <<END
+                       BEGIN;
+                       UPDATE resources
+                               SET owntaskid = $(taskid magic idle)
+                               WHERE owntaskid = $(borrowtaskid $task);
+                       COMMIT;
+END
+       done
+       withtest psql_do <<END
+               DELETE FROM tasks
+                       WHERE type='xdbref' AND refkey='$dbname';
+END
+
+       printf "\n"
+
+       cat <<END
+Test database $dbname now set up.
+export OSSTEST_CONFIG=$test_cfg_setting
+END
+       ;;
+
+#========== DROP ==========
+
+drop)
+       parse_only_suffix "$@"
+
+       dbname
+
+       printf "Dropping %s.\n" "$dbname"
+
+       psql_do <<END
+                SET client_min_messages = WARNING;
+               DROP DATABASE IF EXISTS $dbname;
+               UPDATE resources
+                       SET owntaskid = CAST(tasks.refinfo AS INTEGER)
+                       FROM tasks
+                       WHERE resources.owntaskid = tasks.taskid
+                          AND tasks.type = 'xdbref'
+                         AND tasks.refkey = '$dbname'
+                         AND tasks.live
+                         AND tasks.refinfo IS NOT NULL;
+               UPDATE tasks
+                       SET live = 'f'
+                       WHERE tasks.type = 'xdbref'
+                         AND tasks.refkey = '$dbname';
+END
+
+       rm -f $tcfg
+
+       ;;
+
+#========== EPILOGUE ==========
+
+*)
+       fail "unknown operation \`$cmd'"
+       ;;
+esac
-- 
1.7.10.4


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.