diff options
Diffstat (limited to 'integration')
28 files changed, 5364 insertions, 0 deletions
diff --git a/integration/Kyuafile b/integration/Kyuafile new file mode 100644 index 000000000000..2ebb4ec8acca --- /dev/null +++ b/integration/Kyuafile @@ -0,0 +1,16 @@ +syntax(2) + +test_suite("kyua") + +atf_test_program{name="cmd_about_test"} +atf_test_program{name="cmd_config_test"} +atf_test_program{name="cmd_db_exec_test"} +atf_test_program{name="cmd_db_migrate_test"} +atf_test_program{name="cmd_debug_test"} +atf_test_program{name="cmd_help_test"} +atf_test_program{name="cmd_list_test"} +atf_test_program{name="cmd_report_html_test"} +atf_test_program{name="cmd_report_junit_test"} +atf_test_program{name="cmd_report_test"} +atf_test_program{name="cmd_test_test"} +atf_test_program{name="global_test"} diff --git a/integration/Makefile.am.inc b/integration/Makefile.am.inc new file mode 100644 index 000000000000..cf9ad86d5730 --- /dev/null +++ b/integration/Makefile.am.inc @@ -0,0 +1,150 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if WITH_ATF +tests_integrationdir = $(pkgtestsdir)/integration + +tests_integration_DATA = integration/Kyuafile +EXTRA_DIST += $(tests_integration_DATA) + +ATF_SH_BUILD = \ + $(MKDIR_P) integration; \ + echo "\#! $(ATF_SH)" >integration/$${name}; \ + echo "\# AUTOMATICALLY GENERATED FROM Makefile" >>integration/$${name}; \ + if [ -n "$${substs}" ]; then \ + cat $(srcdir)/integration/utils.sh $(srcdir)/integration/$${name}.sh \ + | sed "$${substs}" >>integration/$${name}; \ + else \ + cat $(srcdir)/integration/utils.sh $(srcdir)/integration/$${name}.sh \ + >>integration/$${name}; \ + fi; \ + chmod +x integration/$${name} + +ATF_SH_DEPS = \ + $(srcdir)/integration/utils.sh \ + Makefile + +EXTRA_DIST += integration/utils.sh + +tests_integration_SCRIPTS = integration/cmd_about_test +CLEANFILES += integration/cmd_about_test +EXTRA_DIST += integration/cmd_about_test.sh +integration/cmd_about_test: $(srcdir)/integration/cmd_about_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_about_test"; \ + substs='s,__KYUA_DOCDIR__,$(docdir),g'; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_config_test +CLEANFILES += integration/cmd_config_test +EXTRA_DIST += integration/cmd_config_test.sh +integration/cmd_config_test: $(srcdir)/integration/cmd_config_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_config_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_db_exec_test +CLEANFILES += integration/cmd_db_exec_test +EXTRA_DIST += integration/cmd_db_exec_test.sh +integration/cmd_db_exec_test: $(srcdir)/integration/cmd_db_exec_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_db_exec_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_db_migrate_test +CLEANFILES += integration/cmd_db_migrate_test +EXTRA_DIST += integration/cmd_db_migrate_test.sh +integration/cmd_db_migrate_test: $(srcdir)/integration/cmd_db_migrate_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_db_migrate_test"; \ + substs='s,__KYUA_STOREDIR__,$(storedir),g'; \ + substs="$${substs};s,__KYUA_STORETESTDATADIR__,$(tests_storedir),g"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_debug_test +CLEANFILES += integration/cmd_debug_test +EXTRA_DIST += integration/cmd_debug_test.sh +integration/cmd_debug_test: $(srcdir)/integration/cmd_debug_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_debug_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_help_test +CLEANFILES += integration/cmd_help_test +EXTRA_DIST += integration/cmd_help_test.sh +integration/cmd_help_test: $(srcdir)/integration/cmd_help_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_help_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_list_test +CLEANFILES += integration/cmd_list_test +EXTRA_DIST += integration/cmd_list_test.sh +integration/cmd_list_test: $(srcdir)/integration/cmd_list_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_list_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_report_test +CLEANFILES += integration/cmd_report_test +EXTRA_DIST += integration/cmd_report_test.sh +integration/cmd_report_test: $(srcdir)/integration/cmd_report_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_report_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_report_html_test +CLEANFILES += integration/cmd_report_html_test +EXTRA_DIST += integration/cmd_report_html_test.sh +integration/cmd_report_html_test: \ + $(srcdir)/integration/cmd_report_html_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_report_html_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_report_junit_test +CLEANFILES += integration/cmd_report_junit_test +EXTRA_DIST += integration/cmd_report_junit_test.sh +integration/cmd_report_junit_test: \ + $(srcdir)/integration/cmd_report_junit_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_report_junit_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_test_test +CLEANFILES += integration/cmd_test_test +EXTRA_DIST += integration/cmd_test_test.sh +integration/cmd_test_test: $(srcdir)/integration/cmd_test_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_test_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/global_test +CLEANFILES += integration/global_test +EXTRA_DIST += integration/global_test.sh +integration/global_test: $(srcdir)/integration/global_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="global_test"; \ + $(ATF_SH_BUILD) +endif + +include integration/helpers/Makefile.am.inc diff --git a/integration/cmd_about_test.sh b/integration/cmd_about_test.sh new file mode 100755 index 000000000000..06d5da5ac4c2 --- /dev/null +++ b/integration/cmd_about_test.sh @@ -0,0 +1,158 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Location of installed documents. Used to validate the output of the about +# messages against the golden files. +: "${KYUA_DOCDIR:=__KYUA_DOCDIR__}" + + +# Common code to validate the output of all about information. +# +# \param file The name of the file with the output. +check_all() { + local file="${1}"; shift + + grep -E 'kyua .*[0-9]+\.[0-9]+' "${file}" || \ + atf_fail 'No version reported' + grep 'Copyright' "${file}" || atf_fail 'No license reported' + grep '^\*[^<>]*$' "${file}" || atf_fail 'No authors reported' + grep '^\*.*<.*@.*>$' "${file}" || atf_fail 'No contributors reported' + grep 'Homepage' "${file}" || atf_fail 'No homepage reported' +} + + +utils_test_case all_topics__installed +all_topics__installed_head() { + atf_set "require.files" "${KYUA_DOCDIR}/AUTHORS" \ + "${KYUA_DOCDIR}/CONTRIBUTORS" "${KYUA_DOCDIR}/LICENSE" +} +all_topics__installed_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua about + check_all stdout +} + + +utils_test_case all_topics__override +all_topics__override_body() { + mkdir docs + echo "* Author (no email)" >docs/AUTHORS + echo "* Contributor <contributor@example.net>" >docs/CONTRIBUTORS + echo "Copyright text" >docs/LICENSE + export KYUA_DOCDIR=docs + atf_check -s exit:0 -o save:stdout -e empty kyua about + check_all stdout +} + + +utils_test_case topic__authors__installed +topic__authors__installed_head() { + atf_set "require.files" "${KYUA_DOCDIR}/AUTHORS" \ + "${KYUA_DOCDIR}/CONTRIBUTORS" +} +topic__authors__installed_body() { + grep -h '^\* ' "${KYUA_DOCDIR}/AUTHORS" "${KYUA_DOCDIR}/CONTRIBUTORS" \ + >expout + atf_check -s exit:0 -o file:expout -e empty kyua about authors +} + + +utils_test_case topic__authors__override +topic__authors__override_body() { + mkdir docs + echo "* Author (no email)" >docs/AUTHORS + echo "* Contributor <contributor@example.net>" >docs/CONTRIBUTORS + export KYUA_DOCDIR=docs + cat docs/AUTHORS docs/CONTRIBUTORS >expout + atf_check -s exit:0 -o file:expout -e empty kyua about authors +} + + +utils_test_case topic__license__installed +topic__license__installed_head() { + atf_set "require.files" "${KYUA_DOCDIR}/LICENSE" +} +topic__license__installed_body() { + atf_check -s exit:0 -o file:"${KYUA_DOCDIR}/LICENSE" -e empty \ + kyua about license +} + + +utils_test_case topic__license__override +topic__license__override_body() { + mkdir docs + echo "Copyright text" >docs/LICENSE + export KYUA_DOCDIR=docs + atf_check -s exit:0 -o file:docs/LICENSE -e empty kyua about license +} + + +utils_test_case topic__version +topic__version_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua about version + + local lines="$(wc -l stdout | awk '{ print $1 }')" + [ "${lines}" -eq 1 ] || atf_fail "Version query returned more than one line" + + grep -E '^kyua (.*) [0-9]+\.[0-9]+$' stdout || \ + atf_fail "Invalid version message" +} + + +utils_test_case topic__invalid +topic__invalid_body() { + cat >experr <<EOF +Usage error for command about: Invalid about topic 'foo'. +Type 'kyua help about' for usage information. +EOF + atf_check -s exit:3 -o empty -e file:experr kyua about foo +} + + +utils_test_case too_many_arguments +too_many_arguments_body() { + cat >stderr <<EOF +Usage error for command about: Too many arguments. +Type 'kyua help about' for usage information. +EOF + atf_check -s exit:3 -o empty -e file:stderr kyua about abc def +} + + +atf_init_test_cases() { + atf_add_test_case all_topics__installed + atf_add_test_case all_topics__override + atf_add_test_case topic__authors__installed + atf_add_test_case topic__authors__override + atf_add_test_case topic__license__installed + atf_add_test_case topic__license__override + atf_add_test_case topic__version + atf_add_test_case topic__invalid + + atf_add_test_case too_many_arguments +} diff --git a/integration/cmd_config_test.sh b/integration/cmd_config_test.sh new file mode 100755 index 000000000000..ed457e5c4b37 --- /dev/null +++ b/integration/cmd_config_test.sh @@ -0,0 +1,355 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case defaults +defaults_body() { + atf_check -s exit:0 \ + -o match:'^architecture = ' \ + -o match:'^platform = ' \ + kyua config +} + + +utils_test_case all +all_body() { + mkdir "${HOME}/.kyua" + cat >"${HOME}/.kyua/kyua.conf" <<EOF +syntax(2) +architecture = "my-architecture" +parallelism = 256 +platform = "my-platform" +unprivileged_user = "$(id -u -n)" +test_suites.suite1.the_variable = "value1" +test_suites.suite2.the_variable = "value2" +EOF + + cat >expout <<EOF +architecture = my-architecture +parallelism = 256 +platform = my-platform +test_suites.suite1.the_variable = value1 +test_suites.suite2.the_variable = value2 +unprivileged_user = $(id -u -n) +EOF + + atf_check -s exit:0 -o file:expout -e empty kyua config +} + + +utils_test_case one__ok +one__ok_body() { + mkdir "${HOME}/.kyua" + cat >"${HOME}/.kyua/kyua.conf" <<EOF +syntax(2) +test_suites.first.one = 1 +test_suites.first.two = 2 +EOF + + cat >expout <<EOF +test_suites.first.two = 2 +EOF + + atf_check -s exit:0 -o file:expout -e empty kyua config \ + test_suites.first.two +} + + +utils_test_case one__fail +one__fail_body() { + mkdir "${HOME}/.kyua" + cat >"${HOME}/.kyua/kyua.conf" <<EOF +syntax(2) +test_suites.first.one = 1 +test_suites.first.three = 3 +EOF + + cat >experr <<EOF +kyua: W: 'test_suites.first.two' is not defined. +EOF + + atf_check -s exit:1 -o empty -e file:experr kyua config \ + test_suites.first.two +} + + +utils_test_case many__ok +many__ok_body() { + mkdir "${HOME}/.kyua" + cat >"${HOME}/.kyua/kyua.conf" <<EOF +syntax(2) +architecture = "overriden" +unknown_setting = "foo" +test_suites.first.one = 1 +test_suites.first.two = 2 +EOF + + cat >expout <<EOF +architecture = overriden +test_suites.first.two = 2 +test_suites.first.one = 1 +EOF + + atf_check -s exit:0 -o file:expout -e empty kyua config \ + architecture \ + test_suites.first.two \ + test_suites.first.one # Inverse order on purpose. + atf_check -s exit:0 -o match:architecture -o not-match:unknown_setting \ + -e empty kyua config +} + + +utils_test_case many__fail +many__fail_body() { + mkdir "${HOME}/.kyua" + cat >"${HOME}/.kyua/kyua.conf" <<EOF +syntax(2) +test_suites.first.one = 1 +test_suites.first.three = 3 +EOF + + cat >expout <<EOF +test_suites.first.one = 1 +test_suites.first.three = 3 +EOF + + cat >experr <<EOF +kyua: W: 'test_suites.first.two' is not defined. +kyua: W: 'test_suites.first.fourth' is not defined. +EOF + + atf_check -s exit:1 -o file:expout -e file:experr kyua config \ + test_suites.first.one test_suites.first.two \ + test_suites.first.three test_suites.first.fourth +} + + +utils_test_case config_flag__default_system +config_flag__default_system_body() { + cat >kyua.conf <<EOF +syntax(2) +test_suites.foo.var = "baz" +EOF + + atf_check -s exit:1 -o empty \ + -e match:"kyua: W: 'test_suites.foo.var'.*not defined" \ + kyua config test_suites.foo.var + export KYUA_CONFDIR="$(pwd)" + atf_check -s exit:0 -o match:"foo.var = baz" -e empty \ + kyua config test_suites.foo.var +} + + +utils_test_case config_flag__default_home +config_flag__default_home_body() { + cat >kyua.conf <<EOF +syntax(2) +test_suites.foo.var = "bar" +EOF + export KYUA_CONFDIR="$(pwd)" + atf_check -s exit:0 -o match:"test_suites.foo.var = bar" -e empty \ + kyua config test_suites.foo.var + + # The previously-created "system-wide" file has to be ignored. + mkdir .kyua + cat >.kyua/kyua.conf <<EOF +syntax(2) +test_suites.foo.var = "baz" +EOF + atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \ + kyua config test_suites.foo.var +} + + +utils_test_case config_flag__explicit__ok +config_flag__explicit__ok_body() { + cat >kyua.conf <<EOF +syntax(2) +test_suites.foo.var = "baz" +EOF + + atf_check -s exit:1 -o empty \ + -e match:"kyua: W: 'test_suites.foo.var'.*not defined" \ + kyua config test_suites.foo.var + atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \ + kyua -c kyua.conf config test_suites.foo.var + atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \ + kyua --config=kyua.conf config test_suites.foo.var +} + + +utils_test_case config_flag__explicit__disable +config_flag__explicit__disable_body() { + cat >kyua.conf <<EOF +syntax(2) +test_suites.foo.var = "baz" +EOF + mkdir .kyua + cp kyua.conf .kyua/kyua.conf + export KYUA_CONFDIR="$(pwd)" + + atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \ + kyua config test_suites.foo.var + atf_check -s exit:1 -o empty \ + -e match:"kyua: W: 'test_suites.foo.var'.*not defined" \ + kyua --config=none config test_suites.foo.var +} + + +utils_test_case config_flag__explicit__missing_file +config_flag__explicit__missing_file_body() { + cat >experr <<EOF +kyua: E: Load of 'foo' failed: File 'foo' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua --config=foo config +} + + +utils_test_case config_flag__explicit__bad_file +config_flag__explicit__bad_file_body() { + touch custom + atf_check -s exit:2 -o empty -e match:"No syntax defined" \ + kyua --config=custom config +} + + +utils_test_case variable_flag__no_config +variable_flag__no_config_body() { + atf_check -s exit:0 \ + -o match:'test_suites.suite1.the_variable = value1' \ + -o match:'test_suites.suite2.the_variable = value2' \ + -e empty \ + kyua \ + -v "test_suites.suite1.the_variable=value1" \ + -v "test_suites.suite2.the_variable=value2" \ + config + + atf_check -s exit:0 \ + -o match:'test_suites.suite1.the_variable = value1' \ + -o match:'test_suites.suite2.the_variable = value2' \ + -e empty \ + kyua \ + --variable="test_suites.suite1.the_variable=value1" \ + --variable="test_suites.suite2.the_variable=value2" \ + config +} + + +utils_test_case variable_flag__override_default_config +variable_flag__override_default_config_body() { + mkdir "${HOME}/.kyua" + cat >"${HOME}/.kyua/kyua.conf" <<EOF +syntax(2) +test_suites.suite1.the_variable = "value1" +test_suites.suite2.the_variable = "should not be used" +EOF + + atf_check -s exit:0 \ + -o match:'test_suites.suite1.the_variable = value1' \ + -o match:'test_suites.suite2.the_variable = overriden' \ + -o match:'test_suites.suite3.the_variable = new' \ + -e empty kyua \ + -v "test_suites.suite2.the_variable=overriden" \ + -v "test_suites.suite3.the_variable=new" \ + config + + atf_check -s exit:0 \ + -o match:'test_suites.suite1.the_variable = value1' \ + -o match:'test_suites.suite2.the_variable = overriden' \ + -o match:'test_suites.suite3.the_variable = new' \ + -e empty kyua \ + --variable="test_suites.suite2.the_variable=overriden" \ + --variable="test_suites.suite3.the_variable=new" \ + config +} + + +utils_test_case variable_flag__override_custom_config +variable_flag__override_custom_config_body() { + cat >config <<EOF +syntax(2) +test_suites.suite1.the_variable = "value1" +test_suites.suite2.the_variable = "should not be used" +EOF + + atf_check -s exit:0 \ + -o match:'test_suites.suite2.the_variable = overriden' \ + -e empty kyua -c config \ + -v "test_suites.suite2.the_variable=overriden" config + + atf_check -s exit:0 \ + -o match:'test_suites.suite2.the_variable = overriden' \ + -e empty kyua -c config \ + --variable="test_suites.suite2.the_variable=overriden" config +} + + +utils_test_case variable_flag__invalid_key +variable_flag__invalid_key_body() { + # CHECK_STYLE_DISABLE + cat >experr <<EOF +Usage error: Invalid argument '' for option --variable: Argument does not have the form 'K=V'. +Type 'kyua help' for usage information. +EOF + # CHECK_STYLE_ENABLE + atf_check -s exit:3 -o empty -e file:experr kyua \ + -v "test_suites.a.b=c" -v "" config +} + + +utils_test_case variable_flag__invalid_value +variable_flag__invalid_value_body() { + cat >experr <<EOF +kyua: E: Invalid value for property 'parallelism': Must be a positive integer. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua \ + -v "parallelism=0" config +} + + +atf_init_test_cases() { + atf_add_test_case defaults + atf_add_test_case all + atf_add_test_case one__ok + atf_add_test_case one__fail + atf_add_test_case many__ok + atf_add_test_case many__fail + + atf_add_test_case config_flag__default_system + atf_add_test_case config_flag__default_home + atf_add_test_case config_flag__explicit__ok + atf_add_test_case config_flag__explicit__disable + atf_add_test_case config_flag__explicit__missing_file + atf_add_test_case config_flag__explicit__bad_file + + atf_add_test_case variable_flag__no_config + atf_add_test_case variable_flag__override_default_config + atf_add_test_case variable_flag__override_custom_config + atf_add_test_case variable_flag__invalid_key + atf_add_test_case variable_flag__invalid_value +} diff --git a/integration/cmd_db_exec_test.sh b/integration/cmd_db_exec_test.sh new file mode 100755 index 000000000000..c260f23a78e1 --- /dev/null +++ b/integration/cmd_db_exec_test.sh @@ -0,0 +1,165 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Creates a new database file in the store directory. +# +# Subsequent invocations of db-exec should just pick this new file up. +create_empty_store() { + cat >Kyuafile <<EOF +syntax(2) +EOF + atf_check -s exit:0 -o ignore -e empty kyua test +} + + +utils_test_case one_arg +one_arg_body() { + create_empty_store + + atf_check -s exit:0 -o save:metadata.csv -e empty \ + kyua db-exec "SELECT * FROM metadata" + atf_check -s exit:0 -o ignore -e empty \ + grep 'schema_version,.*timestamp' metadata.csv +} + + +utils_test_case many_args +many_args_body() { + create_empty_store + + atf_check -s exit:0 -o save:metadata.csv -e empty \ + kyua db-exec SELECT "*" FROM metadata + atf_check -s exit:0 -o ignore -e empty \ + grep 'schema_version,.*timestamp' metadata.csv +} + + +utils_test_case no_args +no_args_body() { + atf_check -s exit:3 -o empty -e match:"Not enough arguments" kyua db-exec + test ! -d .kyua/store/ || atf_fail "Database created but it should" \ + "not have been" +} + + +utils_test_case invalid_statement +invalid_statement_body() { + create_empty_store + + atf_check -s exit:1 -o empty -e match:"SQLite error.*foo" \ + kyua db-exec foo +} + + +utils_test_case no_create_store +no_create_store_body() { + atf_check -s exit:1 -o empty -e match:"No previous results.*not-here" \ + kyua db-exec --results-file=not-here "SELECT * FROM metadata" + if [ -f not-here ]; then + atf_fail "Database created but it should not have been" + fi +} + + +utils_test_case results_file__default_home +results_file__default_home_body() { + HOME=home-dir + create_empty_store + + atf_check -s exit:0 -o save:metadata.csv -e empty \ + kyua db-exec "SELECT * FROM metadata" + test -f home-dir/.kyua/store/*.db || atf_fail "Database not created in" \ + "the home directory" + atf_check -s exit:0 -o ignore -e empty \ + grep 'schema_version,.*timestamp' metadata.csv +} + + +utils_test_case results_file__explicit__ok +results_file__explicit__ok_body() { + create_empty_store + mv .kyua/store/*.db custom.db + rmdir .kyua/store + + HOME=home-dir + atf_check -s exit:0 -o save:metadata.csv -e empty \ + kyua --logfile=/dev/null db-exec -r custom.db "SELECT * FROM metadata" + test ! -d home-dir/.kyua || atf_fail "Home directory created but this" \ + "should not have happened" + atf_check -s exit:0 -o ignore -e empty \ + grep 'schema_version,.*timestamp' metadata.csv +} + + +utils_test_case results_file__explicit__fail +results_file__explicit__fail_head() { + atf_set "require.user" "unprivileged" +} +results_file__explicit__fail_body() { + atf_check -s exit:1 -o empty -e match:"No previous results.*foo.db" \ + kyua db-exec --results-file=foo.db "SELECT * FROM metadata" +} + + +utils_test_case no_headers_flag +no_headers_flag_body() { + create_empty_store + + atf_check kyua db-exec "CREATE TABLE data" \ + "(a INTEGER PRIMARY KEY, b INTEGER, c TEXT)" + atf_check kyua db-exec "INSERT INTO data VALUES (65, 43, NULL)" + atf_check kyua db-exec "INSERT INTO data VALUES (23, 42, 'foo')" + + cat >expout <<EOF +a,b,c +23,42,foo +65,43,NULL +EOF + atf_check -s exit:0 -o file:expout -e empty \ + kyua db-exec "SELECT * FROM data ORDER BY a" + + tail -n 2 <expout >expout2 + atf_check -s exit:0 -o file:expout2 -e empty \ + kyua db-exec --no-headers "SELECT * FROM data ORDER BY a" +} + + +atf_init_test_cases() { + atf_add_test_case one_arg + atf_add_test_case many_args + atf_add_test_case no_args + atf_add_test_case invalid_statement + atf_add_test_case no_create_store + + atf_add_test_case results_file__default_home + atf_add_test_case results_file__explicit__ok + atf_add_test_case results_file__explicit__fail + + atf_add_test_case no_headers_flag +} diff --git a/integration/cmd_db_migrate_test.sh b/integration/cmd_db_migrate_test.sh new file mode 100755 index 000000000000..404a4e774019 --- /dev/null +++ b/integration/cmd_db_migrate_test.sh @@ -0,0 +1,167 @@ +# Copyright 2013 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Location of installed schema files. +: "${KYUA_STOREDIR:=__KYUA_STOREDIR__}" + + +# Location of installed test data files. +: "${KYUA_STORETESTDATADIR:=__KYUA_STORETESTDATADIR__}" + + +# Creates an empty old-style action database. +# +# \param ... Files that contain SQL commands to be run. +create_historical_db() { + mkdir -p "${HOME}/.kyua" + cat "${@}" | sqlite3 "${HOME}/.kyua/store.db" +} + + +# Creates an empty results file. +# +# \param ... Files that contain SQL commands to be run. +create_results_file() { + mkdir -p "${HOME}/.kyua/store" + local dbname="results.$(utils_test_suite_id)-20140718-173200-123456.db" + cat "${@}" | sqlite3 "${HOME}/.kyua/store/${dbname}" +} + + +utils_test_case upgrade__from_v1 +upgrade__from_v1_head() { + atf_set require.files \ + "${KYUA_STORETESTDATADIR}/schema_v1.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v1.sql" \ + "${KYUA_STOREDIR}/migrate_v1_v2.sql" \ + "${KYUA_STOREDIR}/migrate_v2_v3.sql" + atf_set require.progs "sqlite3" +} +upgrade__from_v1_body() { + create_historical_db "${KYUA_STORETESTDATADIR}/schema_v1.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v1.sql" + atf_check -s exit:0 -o empty -e empty kyua db-migrate + for f in \ + "results.test_suite_root.20130108-111331-000000.db" \ + "results.usr_tests.20130108-123832-000000.db" \ + "results.usr_tests.20130108-112635-000000.db" + do + [ -f "${HOME}/.kyua/store/${f}" ] || atf_fail "Expected file ${f}" \ + "was not created" + done + [ ! -f "${HOME}/.kyua/store.db" ] || atf_fail "Historical database not" \ + "deleted" +} + + +utils_test_case upgrade__from_v2 +upgrade__from_v2_head() { + atf_set require.files \ + "${KYUA_STORETESTDATADIR}/schema_v2.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v2.sql" \ + "${KYUA_STOREDIR}/migrate_v2_v3.sql" + atf_set require.progs "sqlite3" +} +upgrade__from_v2_body() { + create_historical_db "${KYUA_STORETESTDATADIR}/schema_v2.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v2.sql" + atf_check -s exit:0 -o empty -e empty kyua db-migrate + for f in \ + "results.test_suite_root.20130108-111331-000000.db" \ + "results.usr_tests.20130108-123832-000000.db" \ + "results.usr_tests.20130108-112635-000000.db" + do + [ -f "${HOME}/.kyua/store/${f}" ] || atf_fail "Expected file ${f}" \ + "was not created" + done + [ ! -f "${HOME}/.kyua/store.db" ] || atf_fail "Historical database not" \ + "deleted" +} + + +utils_test_case already_up_to_date +already_up_to_date_head() { + atf_set require.files "${KYUA_STOREDIR}/schema_v3.sql" + atf_set require.progs "sqlite3" +} +already_up_to_date_body() { + create_results_file "${KYUA_STOREDIR}/schema_v3.sql" + atf_check -s exit:1 -o empty -e match:"already at schema version" \ + kyua db-migrate +} + + +utils_test_case need_upgrade +need_upgrade_head() { + atf_set require.files "${KYUA_STORETESTDATADIR}/schema_v1.sql" + atf_set require.progs "sqlite3" +} +need_upgrade_body() { + create_results_file "${KYUA_STORETESTDATADIR}/schema_v1.sql" + atf_check -s exit:2 -o empty \ + -e match:"database has schema version 1.*use db-migrate" kyua report +} + + +utils_test_case results_file__ok +results_file__ok_body() { + echo "This is not a valid database" >test.db + atf_check -s exit:1 -o empty -e match:"Migration failed" \ + kyua db-migrate --results-file ./test.db +} + + +utils_test_case results_file__fail +results_file__fail_body() { + atf_check -s exit:1 -o empty -e match:"No previous results.*test.db" \ + kyua db-migrate --results-file ./test.db +} + + +utils_test_case too_many_arguments +too_many_arguments_body() { + cat >stderr <<EOF +Usage error for command db-migrate: Too many arguments. +Type 'kyua help db-migrate' for usage information. +EOF + atf_check -s exit:3 -o empty -e file:stderr kyua db-migrate abc def +} + + +atf_init_test_cases() { + atf_add_test_case upgrade__from_v1 + atf_add_test_case upgrade__from_v2 + atf_add_test_case already_up_to_date + atf_add_test_case need_upgrade + + atf_add_test_case results_file__ok + atf_add_test_case results_file__fail + + atf_add_test_case too_many_arguments +} diff --git a/integration/cmd_debug_test.sh b/integration/cmd_debug_test.sh new file mode 100755 index 000000000000..b34a96d72eda --- /dev/null +++ b/integration/cmd_debug_test.sh @@ -0,0 +1,421 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case no_args +no_args_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="simple_all_pass"} +EOF + utils_cp_helper simple_all_pass . + + cat >experr <<EOF +Usage error for command debug: Not enough arguments. +Type 'kyua help debug' for usage information. +EOF + atf_check -s exit:3 -o empty -e file:experr kyua debug +} + + +utils_test_case many_args +many_args_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + + cat >experr <<EOF +Usage error for command debug: Too many arguments. +Type 'kyua help debug' for usage information. +EOF + atf_check -s exit:3 -o empty -e file:experr kyua debug first:pass \ + second:pass +} + + +utils_test_case one_arg__ok_pass +one_arg__ok_pass_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper expect_all_pass first + utils_cp_helper simple_all_pass second + + cat >expout <<EOF +This is the stdout of pass +second:pass -> passed +EOF +cat >experr <<EOF +This is the stderr of pass +EOF + atf_check -s exit:0 -o file:expout -e file:experr kyua debug second:pass +} + + +utils_test_case one_arg__ok_fail +one_arg__ok_fail_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +EOF + utils_cp_helper simple_some_fail first + + cat >expout <<EOF +This is the stdout of fail +first:fail -> failed: This fails on purpose +EOF + cat >experr <<EOF +This is the stderr of fail +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua debug first:fail +} + + +utils_test_case one_arg__no_match +one_arg__no_match_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper expect_all_pass first + utils_cp_helper simple_all_pass second + + cat >experr <<EOF +kyua: E: Unknown test case 'second:die'. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua debug second:die +} + + +utils_test_case one_arg__no_test_case +one_arg__no_test_case_body() { + # CHECK_STYLE_DISABLE + cat >experr <<EOF +Usage error for command debug: 'foo' is not a test case identifier (missing ':'?). +Type 'kyua help debug' for usage information. +EOF + # CHECK_STYLE_ENABLE + atf_check -s exit:3 -o empty -e file:experr kyua debug foo +} + + +utils_test_case one_arg__bad_filter +one_arg__bad_filter_body() { + cat >experr <<EOF +kyua: E: Test case component in 'foo:' is empty. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua debug foo: +} + + +utils_test_case body_and_cleanup +body_and_cleanup_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="single"} +EOF + utils_cp_helper metadata single + + cat >expout <<EOF +single:with_cleanup -> passed +EOF + atf_check -s exit:0 -o file:expout -e empty kyua debug \ + --stdout=saved.out --stderr=saved.err single:with_cleanup + + cat >expout <<EOF +Body message to stdout +Cleanup message to stdout +EOF + atf_check -s exit:0 -o file:expout -e empty cat saved.out + + cat >experr <<EOF +Body message to stderr +Cleanup message to stderr +EOF + atf_check -s exit:0 -o file:experr -e empty cat saved.err +} + + +utils_test_case stdout_stderr_flags +stdout_stderr_flags_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper expect_all_pass first + utils_cp_helper simple_all_pass second + + cat >expout <<EOF +second:pass -> passed +EOF + atf_check -s exit:0 -o file:expout -e empty kyua debug \ + --stdout=saved.out --stderr=saved.err second:pass + + cat >expout <<EOF +This is the stdout of pass +EOF + cmp -s saved.out expout || atf_fail "--stdout did not redirect the" \ + "standard output to the desired file" + + cat >experr <<EOF +This is the stderr of pass +EOF + cmp -s saved.err experr || atf_fail "--stderr did not redirect the" \ + "standard error to the desired file" +} + + +utils_test_case args_are_relative +args_are_relative_body() { + mkdir root + cat >root/Kyuafile <<EOF +syntax(2) +test_suite("integration") +include("subdir/Kyuafile") +atf_test_program{name="prog"} +EOF + utils_cp_helper simple_all_pass root/prog + + mkdir root/subdir + cat >root/subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="prog"} +EOF + utils_cp_helper simple_some_fail root/subdir/prog + + cat >expout <<EOF +This is the stdout of fail +subdir/prog:fail -> failed: This fails on purpose +EOF + cat >experr <<EOF +This is the stderr of fail +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua debug \ + -k "$(pwd)/root/Kyuafile" subdir/prog:fail +} + + +utils_test_case only_load_used_test_programs +only_load_used_test_programs_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper bad_test_program second + + cat >expout <<EOF +This is the stdout of pass +first:pass -> passed +EOF + cat >experr <<EOF +This is the stderr of pass +EOF + CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE + atf_check -s exit:0 -o file:expout -e file:experr kyua debug first:pass + if [ -f "${CREATE_COOKIE}" ]; then + atf_fail "An unmatched test case has been executed, which harms" \ + "performance" + fi +} + + +utils_test_case config_behavior +config_behavior_body() { + cat >"my-config" <<EOF +syntax(2) +test_suites.suite1["the-variable"] = "value1" +test_suites.suite2["the-variable"] = "override me" +EOF + + cat >Kyuafile <<EOF +syntax(2) +atf_test_program{name="config1", test_suite="suite1"} +atf_test_program{name="config2", test_suite="suite2"} +atf_test_program{name="config3", test_suite="suite3"} +EOF + utils_cp_helper config config1 + utils_cp_helper config config2 + utils_cp_helper config config3 + + atf_check -s exit:1 -o match:'failed' -e empty \ + kyua -c my-config -v test_suites.suite2.the-variable=value2 \ + debug config1:get_variable + atf_check -s exit:0 -o match:'passed' -e empty \ + kyua -c my-config -v test_suites.suite2.the-variable=value2 \ + debug config2:get_variable + atf_check -s exit:0 -o match:'skipped' -e empty \ + kyua -c my-config -v test_suites.suite2.the-variable=value2 \ + debug config3:get_variable +} + + +utils_test_case build_root_flag +build_root_flag_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + mkdir build + utils_cp_helper expect_all_pass build/first + utils_cp_helper simple_all_pass build/second + + cat >expout <<EOF +This is the stdout of pass +second:pass -> passed +EOF +cat >experr <<EOF +This is the stderr of pass +EOF + atf_check -s exit:0 -o file:expout -e file:experr \ + kyua debug --build-root=build second:pass +} + + +utils_test_case kyuafile_flag__ok +kyuafile_flag__ok_body() { + cat >Kyuafile <<EOF +This file is bogus but it is not loaded. +EOF + + cat >myfile <<EOF +syntax(2) +test_suite("hello-world") +atf_test_program{name="sometest"} +EOF + utils_cp_helper simple_all_pass sometest + + atf_check -s exit:0 -o match:passed -e empty kyua test -k myfile sometest + atf_check -s exit:0 -o match:passed -e empty kyua test --kyuafile=myfile \ + sometest +} + + +utils_test_case missing_kyuafile +missing_kyuafile_body() { + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua debug foo:bar +} + + +utils_test_case bogus_kyuafile +bogus_kyuafile_body() { + cat >Kyuafile <<EOF +Hello, world. +EOF + atf_check -s exit:2 -o empty \ + -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list +} + + +utils_test_case bogus_test_program +bogus_test_program_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="crash_on_list"} +atf_test_program{name="non_executable"} +EOF + utils_cp_helper bad_test_program crash_on_list + echo 'I am not executable' >non_executable + + cat >experr <<EOF +kyua: E: Unknown test case 'crash_on_list:a'. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua debug crash_on_list:a + + cat >experr <<EOF +kyua: E: Unknown test case 'non_executable:a'. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua debug non_executable:a + + # CHECK_STYLE_DISABLE + cat >expout <<EOF +crash_on_list:__test_cases_list__ -> broken: Invalid header for test case list; expecting Content-Type for application/X-atf-tp version 1, got '' +EOF + # CHECK_STYLE_ENABLE + atf_check -s exit:1 -o file:expout -e empty kyua debug \ + crash_on_list:__test_cases_list__ + + # CHECK_STYLE_DISABLE + cat >expout <<EOF +non_executable:__test_cases_list__ -> broken: Permission denied to run test program +EOF + # CHECK_STYLE_ENABLE + atf_check -s exit:1 -o file:expout -e empty kyua debug \ + non_executable:__test_cases_list__ +} + + +atf_init_test_cases() { + atf_add_test_case no_args + atf_add_test_case many_args + atf_add_test_case one_arg__ok_pass + atf_add_test_case one_arg__ok_fail + atf_add_test_case one_arg__no_match + atf_add_test_case one_arg__no_test_case + atf_add_test_case one_arg__bad_filter + + atf_add_test_case body_and_cleanup + + atf_add_test_case stdout_stderr_flags + + atf_add_test_case args_are_relative + + atf_add_test_case only_load_used_test_programs + + atf_add_test_case config_behavior + + atf_add_test_case build_root_flag + atf_add_test_case kyuafile_flag__ok + atf_add_test_case missing_kyuafile + atf_add_test_case bogus_kyuafile + atf_add_test_case bogus_test_program +} diff --git a/integration/cmd_help_test.sh b/integration/cmd_help_test.sh new file mode 100755 index 000000000000..d8afbd0e6aba --- /dev/null +++ b/integration/cmd_help_test.sh @@ -0,0 +1,93 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case global +global_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua help + grep -E 'kyua .*[0-9]+\.[0-9]+' stdout || atf_fail 'No version reported' + grep '^Usage: kyua' stdout || atf_fail 'No usage line printed' + grep -- '--loglevel' stdout || atf_fail 'Generic options not printed' + if grep -- '--show' stdout; then + atf_fail 'One option of the about subcommand appeared in the output' + fi + grep 'about *Shows detailed' stdout || atf_fail 'Commands not printed' +} + + +utils_test_case one_command +one_command_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua help test + grep -E 'kyua .*[0-9]+\.[0-9]+' stdout || atf_fail 'No version reported' + grep '^Usage: kyua' stdout || atf_fail 'No usage line printed' + grep '^Run tests' stdout || atf_fail 'No description printed' + grep -- '--loglevel' stdout || atf_fail 'Generic options not printed' + grep -- '--kyuafile' stdout || atf_fail 'Command options not printed' + if grep 'about: Shows detailed' stdout; then + atf_fail 'Printed table of commands, but should not have done so' + fi +} + + +utils_test_case ignore_bad_config +ignore_bad_config_body() { + echo 'this is an invalid configuration file' >bad-config + atf_check -s exit:0 -o save:stdout -e empty kyua -c bad-config help + grep '^Usage: kyua' stdout || atf_fail 'No usage line printed' + grep -- '--loglevel' stdout || atf_fail 'Generic options not printed' +} + + +utils_test_case unknown_command +unknown_command_body() { + cat >stderr <<EOF +Usage error for command help: The command abc does not exist. +Type 'kyua help help' for usage information. +EOF + atf_check -s exit:3 -o empty -e file:stderr kyua help abc +} + + +utils_test_case too_many_arguments +too_many_arguments_body() { + cat >stderr <<EOF +Usage error for command help: Too many arguments. +Type 'kyua help help' for usage information. +EOF + atf_check -s exit:3 -o empty -e file:stderr kyua help about cde +} + + +atf_init_test_cases() { + atf_add_test_case global + atf_add_test_case one_command + + atf_add_test_case ignore_bad_config + atf_add_test_case unknown_command + atf_add_test_case too_many_arguments +} diff --git a/integration/cmd_list_test.sh b/integration/cmd_list_test.sh new file mode 100755 index 000000000000..a916e0f2ec4b --- /dev/null +++ b/integration/cmd_list_test.sh @@ -0,0 +1,600 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case no_args +no_args_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="metadata"} +atf_test_program{name="simple_all_pass"} +include("subdir/Kyuafile") +EOF + utils_cp_helper metadata . + utils_cp_helper simple_all_pass . + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration2") +atf_test_program{name="simple_some_fail"} +EOF + utils_cp_helper simple_some_fail subdir + + cat >expout <<EOF +metadata:many_properties +metadata:no_properties +metadata:one_property +metadata:with_cleanup +simple_all_pass:pass +simple_all_pass:skip +subdir/simple_some_fail:fail +subdir/simple_some_fail:pass +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list +} + + +utils_test_case one_arg__subdir +one_arg__subdir_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +include("subdir/Kyuafile") +EOF + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("in-subdir") +atf_test_program{name="simple_all_pass"} +EOF + utils_cp_helper simple_all_pass subdir + + cat >expout <<EOF +subdir/simple_all_pass:pass +subdir/simple_all_pass:skip +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list subdir +} + + +utils_test_case one_arg__test_case +one_arg__test_case_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + + cat >expout <<EOF +first:skip +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list first:skip +} + + +utils_test_case one_arg__test_program +one_arg__test_program_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_some_fail second + + cat >expout <<EOF +second:fail +second:pass +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list second +} + + +utils_test_case one_arg__invalid +one_arg__invalid_body() { +cat >experr <<EOF +kyua: E: Test case component in 'foo:' is empty. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua list foo: + +cat >experr <<EOF +kyua: E: Program name '/a/b' must be relative to the test suite, not absolute. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua list /a/b +} + + +utils_test_case many_args__ok +many_args__ok_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +include("subdir/Kyuafile") +atf_test_program{name="first"} +EOF + utils_cp_helper simple_all_pass first + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("in-subdir") +atf_test_program{name="second"} +EOF + utils_cp_helper simple_some_fail subdir/second + + cat >expout <<EOF +subdir/second:fail (in-subdir) +subdir/second:pass (in-subdir) +first:pass (top-level) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list -v subdir first:pass +} + + +utils_test_case many_args__invalid +many_args__invalid_body() { +cat >experr <<EOF +kyua: E: Program name component in ':badbad' is empty. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok :badbad + +cat >experr <<EOF +kyua: E: Program name '/foo' must be relative to the test suite, not absolute. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok /foo +} + + +utils_test_case many_args__no_match__all +many_args__no_match__all_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'first1'. +EOF + atf_check -s exit:1 -o empty -e file:experr kyua list first1 +} + + +utils_test_case many_args__no_match__some +many_args__no_match__some_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +atf_test_program{name="third"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + utils_cp_helper simple_some_fail third + + cat >expout <<EOF +first:pass +first:skip +third:fail +third:pass +EOF + + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'fifth'. +kyua: W: No test cases matched by the filter 'fourth'. +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua list first fourth \ + third fifth +} + + +utils_test_case args_are_relative +args_are_relative_body() { + mkdir root + cat >root/Kyuafile <<EOF +syntax(2) +test_suite("integration-1") +atf_test_program{name="first"} +atf_test_program{name="second"} +include("subdir/Kyuafile") +EOF + utils_cp_helper simple_all_pass root/first + utils_cp_helper simple_some_fail root/second + + mkdir root/subdir + cat >root/subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration-2") +atf_test_program{name="third"} +atf_test_program{name="fourth"} +EOF + utils_cp_helper simple_all_pass root/subdir/third + utils_cp_helper simple_some_fail root/subdir/fourth + + cat >expout <<EOF +first:pass (integration-1) +first:skip (integration-1) +subdir/fourth:fail (integration-2) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list \ + -v -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail +} + + +utils_test_case only_load_used_test_programs +only_load_used_test_programs_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper bad_test_program second + + cat >expout <<EOF +first:pass +first:skip +EOF + CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE + atf_check -s exit:0 -o file:expout -e empty kyua list first + if [ -f "${CREATE_COOKIE}" ]; then + atf_fail "An unmatched test case has been executed, which harms" \ + "performance" + fi +} + + +utils_test_case config_behavior +config_behavior_body() { + cat >"my-config" <<EOF +syntax(2) +test_suites.suite1["the-variable"] = "value1" +EOF + + cat >Kyuafile <<EOF +syntax(2) +atf_test_program{name="config1", test_suite="suite1"} +EOF + utils_cp_helper config config1 + + CONFIG_VAR_FILE="$(pwd)/cookie"; export CONFIG_VAR_FILE + if [ -f "${CONFIG_VAR_FILE}" ]; then + atf_fail "Cookie file already created; test case list may have gotten" \ + "a bad configuration" + fi + atf_check -s exit:0 -o ignore -e empty kyua -c my-config list + [ -f "${CONFIG_VAR_FILE}" ] || \ + atf_fail "Cookie file not created; test case list did not get" \ + "configuration variables" + value="$(cat "${CONFIG_VAR_FILE}")" + [ "${value}" = "value1" ] || \ + atf_fail "Invalid value (${value}) in cookie file; test case list did" \ + "not get the correct configuration variables" +} + + +utils_test_case build_root_flag +build_root_flag_body() { + mkdir subdir + mkdir build + mkdir build/subdir + + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +include("subdir/Kyuafile") +atf_test_program{name="first"} +EOF + echo 'invalid' >first + utils_cp_helper simple_all_pass build/first + + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("in-subdir") +atf_test_program{name="second"} +EOF + echo 'invalid' >subdir/second + utils_cp_helper simple_some_fail build/subdir/second + + cat >expout <<EOF +subdir/second:fail +subdir/second:pass +first:pass +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list --build-root=build \ + subdir first:pass +} + + +utils_test_case kyuafile_flag__no_args +kyuafile_flag__no_args_body() { + cat >Kyuafile <<EOF +This file is bogus but it is not loaded. +EOF + + cat >myfile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="sometest"} +EOF + utils_cp_helper simple_all_pass sometest + + cat >expout <<EOF +sometest:pass +sometest:skip +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list -k myfile + atf_check -s exit:0 -o file:expout -e empty kyua list --kyuafile=myfile +} + + +utils_test_case kyuafile_flag__some_args +kyuafile_flag__some_args_body() { + cat >Kyuafile <<EOF +This file is bogus but it is not loaded. +EOF + + cat >myfile <<EOF +syntax(2) +test_suite("hello-world") +atf_test_program{name="sometest"} +EOF + utils_cp_helper simple_all_pass sometest + + cat >expout <<EOF +sometest:pass (hello-world) +sometest:skip (hello-world) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list -v -k myfile sometest + atf_check -s exit:0 -o file:expout -e empty kyua list -v --kyuafile=myfile \ + sometest +} + + +utils_test_case verbose_flag +verbose_flag_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration-suite-1") +atf_test_program{name="simple_all_pass"} +plain_test_program{name="i_am_plain", timeout=654} +include("subdir/Kyuafile") +EOF + utils_cp_helper simple_all_pass . + touch i_am_plain + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration-suite-2") +atf_test_program{name="metadata"} +EOF + utils_cp_helper metadata subdir + + cat >expout <<EOF +simple_all_pass:pass (integration-suite-1) +simple_all_pass:skip (integration-suite-1) +i_am_plain:main (integration-suite-1) + timeout = 654 +subdir/metadata:many_properties (integration-suite-2) + allowed_architectures = some-architecture + allowed_platforms = some-platform + custom.no-meaning = I am a custom variable + description = A description with some padding + required_configs = var1 var2 var3 + required_files = /my/file1 /some/other/file + required_programs = /nonexistent/bin3 bin1 bin2 + required_user = root +subdir/metadata:no_properties (integration-suite-2) +subdir/metadata:one_property (integration-suite-2) + description = Does nothing but has one metadata property +subdir/metadata:with_cleanup (integration-suite-2) + has_cleanup = true + timeout = 250 +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list -v + atf_check -s exit:0 -o file:expout -e empty kyua list --verbose +} + + +utils_test_case no_test_program_match +no_test_program_match_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'second'. +EOF + atf_check -s exit:1 -o empty -e file:experr kyua list second +} + + +utils_test_case no_test_case_match +no_test_case_match_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +EOF + utils_cp_helper simple_all_pass first + + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'first:foobar'. +EOF + atf_check -s exit:1 -o empty -e file:experr kyua list first:foobar +} + + +utils_test_case missing_kyuafile__no_args +missing_kyuafile__no_args_body() { + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua list +} + + +utils_test_case missing_kyuafile__test_program +missing_kyuafile__test_program_body() { + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="unused"} +EOF + utils_cp_helper simple_all_pass subdir/unused + + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua list subdir/unused +} + + +utils_test_case missing_kyuafile__subdir +missing_kyuafile__subdir_body() { + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="unused"} +EOF + utils_cp_helper simple_all_pass subdir/unused + + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua list subdir +} + + +utils_test_case bogus_kyuafile +bogus_kyuafile_body() { + cat >Kyuafile <<EOF +Hello, world. +EOF + atf_check -s exit:2 -o empty \ + -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list +} + + +utils_test_case bogus_test_program +bogus_test_program_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="crash_on_list"} +atf_test_program{name="non_executable"} +EOF + utils_cp_helper bad_test_program crash_on_list + echo 'I am not executable' >non_executable + + cat >expout <<EOF +crash_on_list:__test_cases_list__ +non_executable:__test_cases_list__ +EOF + atf_check -s exit:0 -o file:expout -e empty kyua list +} + + +utils_test_case missing_test_program +missing_test_program_body() { + cat >Kyuafile <<EOF +syntax(2) +include("subdir/Kyuafile") +EOF + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="ok"} +atf_test_program{name="i-am-missing"} +EOF + echo 'I should not be touched because the Kyuafile is bogus' >subdir/ok + +# CHECK_STYLE_DISABLE + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: .*Non-existent test program 'subdir/i-am-missing'. +EOF +# CHECK_STYLE_ENABLE + atf_check -s exit:2 -o empty -e "match:$(cat experr)" kyua list +} + + +atf_init_test_cases() { + atf_add_test_case no_args + atf_add_test_case one_arg__subdir + atf_add_test_case one_arg__test_case + atf_add_test_case one_arg__test_program + atf_add_test_case one_arg__invalid + atf_add_test_case many_args__ok + atf_add_test_case many_args__invalid + atf_add_test_case many_args__no_match__all + atf_add_test_case many_args__no_match__some + + atf_add_test_case args_are_relative + + atf_add_test_case only_load_used_test_programs + + atf_add_test_case config_behavior + + atf_add_test_case build_root_flag + + atf_add_test_case kyuafile_flag__no_args + atf_add_test_case kyuafile_flag__some_args + + atf_add_test_case verbose_flag + + atf_add_test_case no_test_program_match + atf_add_test_case no_test_case_match + + atf_add_test_case missing_kyuafile__no_args + atf_add_test_case missing_kyuafile__test_program + atf_add_test_case missing_kyuafile__subdir + + atf_add_test_case bogus_kyuafile + atf_add_test_case bogus_test_program + atf_add_test_case missing_test_program +} diff --git a/integration/cmd_report_html_test.sh b/integration/cmd_report_html_test.sh new file mode 100755 index 000000000000..9c9b4ba81c86 --- /dev/null +++ b/integration/cmd_report_html_test.sh @@ -0,0 +1,267 @@ +# Copyright 2012 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Executes a mock test suite to generate data in the database. +# +# \param mock_env The value to store in a MOCK variable in the environment. +# Use this to be able to differentiate executions by inspecting the +# context of the output. +# \param dbfile_name File to which to write the path to the generated database +# file. +run_tests() { + local mock_env="${1}"; shift + local dbfile_name="${1}"; shift + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="simple_all_pass"} +atf_test_program{name="simple_some_fail"} +atf_test_program{name="metadata"} +EOF + + utils_cp_helper simple_all_pass . + utils_cp_helper simple_some_fail . + utils_cp_helper metadata . + atf_check -s exit:1 -o save:stdout -e empty env MOCK="${mock_env}" kyua test + grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}" + rm stdout + + # Ensure the results of 'report-html' come from the database. + rm Kyuafile simple_all_pass simple_some_fail metadata +} + + +# Ensure a file has a set of strings. +# +# \param file The name of the file to check. +# \param ... List of strings to check. +check_in_file() { + local file="${1}"; shift + + while [ ${#} -gt 0 ]; do + echo "Checking for presence of '${1}' in ${file}" + if grep "${1}" "${file}" >/dev/null; then + : + else + atf_fail "Test case output not found in HTML page ${file}" + fi + shift + done +} + + +# Ensure a file does not have a set of strings. +# +# \param file The name of the file to check. +# \param ... List of strings to check. +check_not_in_file() { + local file="${1}"; shift + + while [ ${#} -gt 0 ]; do + echo "Checking for lack of '${1}' in ${file}" + if grep "${1}" "${file}" >/dev/null; then + atf_fail "Spurious test case output found in HTML page" + fi + shift + done +} + + +utils_test_case default_behavior__ok +default_behavior__ok_body() { + run_tests "mock1" unused_dbfile_name + + atf_check -s exit:0 -o ignore -e empty kyua report-html + for f in \ + html/index.html \ + html/context.html \ + html/simple_all_pass_skip.html \ + html/simple_some_fail_fail.html + do + test -f "${f}" || atf_fail "Missing ${f}" + done + + atf_check -o match:"2 TESTS FAILING" cat html/index.html + + check_in_file html/simple_all_pass_skip.html \ + "This is the stdout of skip" "This is the stderr of skip" + check_not_in_file html/simple_all_pass_skip.html \ + "This is the stdout of pass" "This is the stderr of pass" \ + "This is the stdout of fail" "This is the stderr of fail" \ + "Test case did not write anything to" + + check_in_file html/simple_some_fail_fail.html \ + "This is the stdout of fail" "This is the stderr of fail" + check_not_in_file html/simple_some_fail_fail.html \ + "This is the stdout of pass" "This is the stderr of pass" \ + "This is the stdout of skip" "This is the stderr of skip" \ + "Test case did not write anything to" + + check_in_file html/metadata_one_property.html \ + "description = Does nothing but has one metadata property" + check_not_in_file html/metadata_one_property.html \ + "allowed_architectures = some-architecture" + + check_in_file html/metadata_many_properties.html \ + "allowed_architectures = some-architecture" + check_not_in_file html/metadata_many_properties.html \ + "description = Does nothing but has one metadata property" +} + + +utils_test_case default_behavior__no_store +default_behavior__no_store_body() { + echo 'kyua: E: No previous results file found for test suite' \ + "$(utils_test_suite_id)." >experr + atf_check -s exit:2 -o empty -e file:experr kyua report-html +} + + +utils_test_case results_file__explicit +results_file__explicit_body() { + run_tests "mock1" dbfile_name1 + run_tests "mock2" dbfile_name2 + + atf_check -s exit:0 -o ignore -e empty kyua report-html \ + --results-file="$(cat dbfile_name1)" + grep "MOCK.*mock1" html/context.html || atf_fail "Invalid context in report" + + rm -rf html + atf_check -s exit:0 -o ignore -e empty kyua report-html \ + --results-file="$(cat dbfile_name2)" + grep "MOCK.*mock2" html/context.html || atf_fail "Invalid context in report" +} + + +utils_test_case results_file__not_found +results_file__not_found_body() { + atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \ + kyua report-html --results-file=foo +} + + +utils_test_case force__yes +force__yes_body() { + run_tests "mock1" unused_dbfile_name + + atf_check -s exit:0 -o ignore -e empty kyua report-html + test -f html/index.html || atf_fail "Expected file not created" + rm html/index.html + atf_check -s exit:0 -o ignore -e empty kyua report-html --force + test -f html/index.html || atf_fail "Expected file not created" +} + + +utils_test_case force__no +force__no_body() { + run_tests "mock1" unused_dbfile_name + + atf_check -s exit:0 -o ignore -e empty kyua report-html + test -f html/index.html || atf_fail "Expected file not created" + rm html/index.html + +cat >experr <<EOF +kyua: E: Output directory 'html' already exists; maybe use --force?. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua report-html + test ! -f html/index.html || atf_fail "Not expected file created" +} + + +utils_test_case output__explicit +output__explicit_body() { + run_tests "mock1" unused_dbfile_name + + mkdir output + atf_check -s exit:0 -o ignore -e empty kyua report-html --output=output/foo + test ! -d html || atf_fail "Not expected directory created" + test -f output/foo/index.html || atf_fail "Expected file not created" +} + + +utils_test_case results_filter__ok +results_filter__ok_body() { + run_tests "mock1" unused_dbfile_name + + atf_check -s exit:0 -o ignore -e empty kyua report-html \ + --results-filter=passed + for f in \ + html/index.html \ + html/context.html \ + html/simple_all_pass_pass.html \ + html/simple_some_fail_pass.html \ + html/metadata_no_properties.html \ + html/metadata_with_cleanup.html + do + test -f "${f}" || atf_fail "Missing ${f}" + done + + atf_check -o match:"2 TESTS FAILING" cat html/index.html + + check_in_file html/simple_all_pass_pass.html \ + "This is the stdout of pass" "This is the stderr of pass" + check_not_in_file html/simple_all_pass_pass.html \ + "This is the stdout of skip" "This is the stderr of skip" \ + "This is the stdout of fail" "This is the stderr of fail" \ + "Test case did not write anything to" + + check_in_file html/simple_some_fail_pass.html \ + "Test case did not write anything to stdout" \ + "Test case did not write anything to stderr" + check_not_in_file html/simple_some_fail_pass.html \ + "This is the stdout of pass" "This is the stderr of pass" \ + "This is the stdout of skip" "This is the stderr of skip" \ + "This is the stdout of fail" "This is the stderr of fail" +} + + +utils_test_case results_filter__invalid +results_filter__invalid_body() { + echo "kyua: E: Unknown result type 'foo-bar'." >experr + atf_check -s exit:2 -o empty -e file:experr kyua report-html \ + --results-filter=passed,foo-bar +} + + +atf_init_test_cases() { + atf_add_test_case default_behavior__ok + atf_add_test_case default_behavior__no_store + + atf_add_test_case results_file__explicit + atf_add_test_case results_file__not_found + + atf_add_test_case force__yes + atf_add_test_case force__no + + atf_add_test_case output__explicit + + atf_add_test_case results_filter__ok + atf_add_test_case results_filter__invalid +} diff --git a/integration/cmd_report_junit_test.sh b/integration/cmd_report_junit_test.sh new file mode 100755 index 000000000000..af1a464f6004 --- /dev/null +++ b/integration/cmd_report_junit_test.sh @@ -0,0 +1,300 @@ +# Copyright 2014 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Executes a mock test suite to generate data in the database. +# +# \param mock_env The value to store in a MOCK variable in the environment. +# Use this to be able to differentiate executions by inspecting the +# context of the output. +# \param dbfile_name File to which to write the path to the generated database +# file. +run_tests() { + local mock_env="${1}"; shift + local dbfile_name="${1}"; shift + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="simple_all_pass"} +EOF + + utils_cp_helper simple_all_pass . + atf_check -s exit:0 -o save:stdout -e empty env MOCK="${mock_env}" kyua test + grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}" + rm stdout + + # Ensure the results of 'report-junit' come from the database. + rm Kyuafile simple_all_pass +} + + +# Removes the contents of a properties tag from stdout. +strip_properties='awk " +BEGIN { skip = 0; } + +/<\/properties>/ { + print \"</properties>\"; + skip = 0; + next; +} + +/<properties>/ { + print \"<properties>\"; + print \"CONTENTS STRIPPED BY TEST\"; + skip = 1; + next; +} + +{ if (!skip) print; }"' + + +utils_test_case default_behavior__ok +default_behavior__ok_body() { + utils_install_times_wrapper + + run_tests "mock1 +this should not be seen +mock1 new line" unused_dbfile_name + + cat >expout <<EOF +<?xml version="1.0" encoding="iso-8859-1"?> +<testsuite> +<properties> +CONTENTS STRIPPED BY TEST +</properties> +<testcase classname="simple_all_pass" name="pass" time="S.UUU"> +<system-out>This is the stdout of pass +</system-out> +<system-err>Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of pass +</system-err> +</testcase> +<testcase classname="simple_all_pass" name="skip" time="S.UUU"> +<skipped/> +<system-out>This is the stdout of skip +</system-out> +<system-err>Skipped result details +---------------------- + +The reason for skipping is this + +Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of skip +</system-err> +</testcase> +</testsuite> +EOF + atf_check -s exit:0 -o file:expout -e empty -x "kyua report-junit" \ + "| ${strip_properties}" +} + + +utils_test_case default_behavior__no_store +default_behavior__no_store_body() { + echo 'kyua: E: No previous results file found for test suite' \ + "$(utils_test_suite_id)." >experr + atf_check -s exit:2 -o empty -e file:experr kyua report-junit +} + + +utils_test_case results_file__explicit +results_file__explicit_body() { + run_tests "mock1" dbfile_name1 + run_tests "mock2" dbfile_name2 + + atf_check -s exit:0 -o match:"MOCK.*mock1" -o not-match:"MOCK.*mock2" \ + -e empty kyua report-junit --results-file="$(cat dbfile_name1)" + atf_check -s exit:0 -o not-match:"MOCK.*mock1" -o match:"MOCK.*mock2" \ + -e empty kyua report-junit --results-file="$(cat dbfile_name2)" +} + + +utils_test_case results_file__not_found +results_file__not_found_body() { + atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \ + kyua report-junit --results-file=foo +} + + +utils_test_case output__explicit +output__explicit_body() { + run_tests unused_mock unused_dbfile_name + + cat >report <<EOF +<?xml version="1.0" encoding="iso-8859-1"?> +<testsuite> +<properties> +CONTENTS STRIPPED BY TEST +</properties> +<testcase classname="simple_all_pass" name="pass" time="S.UUU"> +<system-out>This is the stdout of pass +</system-out> +<system-err>Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of pass +</system-err> +</testcase> +<testcase classname="simple_all_pass" name="skip" time="S.UUU"> +<skipped/> +<system-out>This is the stdout of skip +</system-out> +<system-err>Skipped result details +---------------------- + +The reason for skipping is this + +Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of skip +</system-err> +</testcase> +</testsuite> +EOF + + atf_check -s exit:0 -o file:report -e empty -x kyua report-junit \ + --output=/dev/stdout "| ${strip_properties} | ${utils_strip_times}" + atf_check -s exit:0 -o empty -e save:stderr kyua report-junit \ + --output=/dev/stderr + atf_check -s exit:0 -o file:report -x cat stderr \ + "| ${strip_properties} | ${utils_strip_times}" + + atf_check -s exit:0 -o empty -e empty kyua report-junit \ + --output=my-file + atf_check -s exit:0 -o file:report -x cat my-file \ + "| ${strip_properties} | ${utils_strip_times}" +} + + +atf_init_test_cases() { + atf_add_test_case default_behavior__ok + atf_add_test_case default_behavior__no_store + + atf_add_test_case results_file__explicit + atf_add_test_case results_file__not_found + + atf_add_test_case output__explicit +} diff --git a/integration/cmd_report_test.sh b/integration/cmd_report_test.sh new file mode 100755 index 000000000000..18a5db386dfd --- /dev/null +++ b/integration/cmd_report_test.sh @@ -0,0 +1,381 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Executes a mock test suite to generate data in the database. +# +# \param mock_env The value to store in a MOCK variable in the environment. +# Use this to be able to differentiate executions by inspecting the +# context of the output. +# \param dbfile_name File to which to write the path to the generated database +# file. +run_tests() { + local mock_env="${1}"; shift + local dbfile_name="${1}"; shift + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="simple_all_pass"} +EOF + + utils_cp_helper simple_all_pass . + atf_check -s exit:0 -o save:stdout -e empty env \ + MOCK="${mock_env}" _='fake-value' kyua test + grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}" + rm stdout + + # Ensure the results of 'report' come from the database. + rm Kyuafile simple_all_pass +} + + +utils_test_case default_behavior__ok +default_behavior__ok_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout <<EOF +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report + + run_tests "mock2" dbfile_name2 + + cat >expout <<EOF +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name2) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report +} + + +utils_test_case default_behavior__no_store +default_behavior__no_store_body() { + echo 'kyua: E: No previous results file found for test suite' \ + "$(utils_test_suite_id)." >experr + atf_check -s exit:2 -o empty -e file:experr kyua report +} + + +utils_test_case results_file__explicit +results_file__explicit_body() { + run_tests "mock1" dbfile_name1 + run_tests "mock2" dbfile_name2 + + atf_check -s exit:0 -o match:"MOCK=mock1" -o not-match:"MOCK=mock2" \ + -e empty kyua report --results-file="$(cat dbfile_name1)" \ + --verbose + atf_check -s exit:0 -o not-match:"MOCK=mock1" -o match:"MOCK=mock2" \ + -e empty kyua report --results-file="$(cat dbfile_name2)" \ + --verbose +} + + +utils_test_case results_file__not_found +results_file__not_found_body() { + atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \ + kyua report --results-file=foo +} + + +utils_test_case output__explicit +output__explicit_body() { + run_tests unused_mock dbfile_name + + cat >report <<EOF +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + + atf_check -s exit:0 -o file:report -e empty -x kyua report \ + --output=/dev/stdout "| ${utils_strip_times_but_not_ids}" + atf_check -s exit:0 -o empty -e save:stderr kyua report \ + --output=/dev/stderr + atf_check -s exit:0 -o file:report -x cat stderr \ + "| ${utils_strip_times_but_not_ids}" + + atf_check -s exit:0 -o empty -e empty kyua report \ + --output=my-file + atf_check -s exit:0 -o file:report -x cat my-file \ + "| ${utils_strip_times_but_not_ids}" +} + + +utils_test_case filter__ok +filter__ok_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout <<EOF +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + simple_all_pass:skip +} + + +utils_test_case filter__ok_passed_excluded_by_default +filter__ok_passed_excluded_by_default_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + # Passed results are excluded by default so they are not displayed even if + # requested with a test case filter. This might be somewhat confusing... + cat >expout <<EOF +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + simple_all_pass:pass + cat >expout <<EOF +===> Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter= simple_all_pass:pass +} + + +utils_test_case filter__no_match +filter__no_match_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout <<EOF +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'first'. +kyua: W: No test cases matched by the filter 'simple_all_pass:second'. +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua report \ + first simple_all_pass:skip simple_all_pass:second +} + + +utils_test_case verbose +verbose_body() { + # Switch to the current directory using its physical location and update + # HOME accordingly. Otherwise, the test below where we compare the value + # of HOME in the output might fail if the path to HOME contains a symlink + # (as is the case in OS X when HOME points to the temporary directory.) + local real_cwd="$(pwd -P)" + cd "${real_cwd}" + HOME="${real_cwd}" + + run_tests "mock1 +has multiple lines +and terminates here" dbfile_name + + cat >expout <<EOF +===> Execution context +Current directory: ${real_cwd} +Environment variables: +EOF + # $_ is a bash variable. To keep our tests stable, we override its value + # below to match the hardcoded value in run_tests. + env \ + HOME="${real_cwd}" \ + MOCK="mock1 +has multiple lines +and terminates here" \ + _='fake-value' \ + "$(atf_get_srcdir)/helpers/dump_env" ' ' ' ' >>expout + cat >>expout <<EOF +===> simple_all_pass:skip +Result: skipped: The reason for skipping is this +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Metadata: + allowed_architectures is empty + allowed_platforms is empty + description is empty + has_cleanup = false + is_exclusive = false + required_configs is empty + required_disk_space = 0 + required_files is empty + required_memory = 0 + required_programs is empty + required_user is empty + timeout = 300 + +Standard output: +This is the stdout of skip + +Standard error: +This is the stderr of skip +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty -x kyua report --verbose \ + "| ${utils_strip_times_but_not_ids}" +} + + +utils_test_case results_filter__empty +results_filter__empty_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout <<EOF +===> Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report --results-filter= +} + + +utils_test_case results_filter__one +results_filter__one_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout <<EOF +===> Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter=passed +} + + +utils_test_case results_filter__multiple_all_match +results_filter__multiple_all_match_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout <<EOF +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter=skipped,passed +} + + +utils_test_case results_filter__multiple_some_match +results_filter__multiple_some_match_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout <<EOF +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter=skipped,xfail,broken,failed +} + + +atf_init_test_cases() { + atf_add_test_case default_behavior__ok + atf_add_test_case default_behavior__no_store + + atf_add_test_case results_file__explicit + atf_add_test_case results_file__not_found + + atf_add_test_case filter__ok + atf_add_test_case filter__ok_passed_excluded_by_default + atf_add_test_case filter__no_match + + atf_add_test_case verbose + + atf_add_test_case output__explicit + + atf_add_test_case results_filter__empty + atf_add_test_case results_filter__one + atf_add_test_case results_filter__multiple_all_match + atf_add_test_case results_filter__multiple_some_match +} diff --git a/integration/cmd_test_test.sh b/integration/cmd_test_test.sh new file mode 100755 index 000000000000..bc8c62daf223 --- /dev/null +++ b/integration/cmd_test_test.sh @@ -0,0 +1,1071 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case one_test_program__all_pass +one_test_program__all_pass_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="simple_all_pass"} +EOF + + cat >expout <<EOF +simple_all_pass:pass -> passed [S.UUUs] +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + + utils_cp_helper simple_all_pass . + atf_check -s exit:0 -o file:expout -e empty kyua test +} + + +utils_test_case one_test_program__some_fail +one_test_program__some_fail_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="simple_some_fail"} +EOF + + cat >expout <<EOF +simple_some_fail:fail -> failed: This fails on purpose [S.UUUs] +simple_some_fail:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/2 passed (1 failed) +EOF + + utils_cp_helper simple_some_fail . + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case many_test_programs__all_pass +many_test_programs__all_pass_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +atf_test_program{name="third"} +plain_test_program{name="fourth", required_files="/non-existent/foo"} +EOF + + cat >expout <<EOF +first:pass -> passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +fourth:main -> skipped: Required file '/non-existent/foo' not found [S.UUUs] +second:pass -> passed [S.UUUs] +second:skip -> skipped: The reason for skipping is this [S.UUUs] +third:pass -> passed [S.UUUs] +third:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +7/7 passed (0 failed) +EOF + + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + utils_cp_helper simple_all_pass third + echo "not executed" >fourth; chmod +x fourth + atf_check -s exit:0 -o file:expout -e empty kyua test +} + + +utils_test_case many_test_programs__some_fail +many_test_programs__some_fail_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +atf_test_program{name="third"} +plain_test_program{name="fourth"} +EOF + + cat >expout <<EOF +first:fail -> failed: This fails on purpose [S.UUUs] +first:pass -> passed [S.UUUs] +fourth:main -> failed: Returned non-success exit status 76 [S.UUUs] +second:fail -> failed: This fails on purpose [S.UUUs] +second:pass -> passed [S.UUUs] +third:pass -> passed [S.UUUs] +third:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +4/7 passed (3 failed) +EOF + + utils_cp_helper simple_some_fail first + utils_cp_helper simple_some_fail second + utils_cp_helper simple_all_pass third + echo '#! /bin/sh' >fourth + echo 'exit 76' >>fourth + chmod +x fourth + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case expect__all_pass +expect__all_pass_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="expect_all_pass"} +EOF + +# CHECK_STYLE_DISABLE + cat >expout <<EOF +expect_all_pass:die -> expected_failure: This is the reason for death [S.UUUs] +expect_all_pass:exit -> expected_failure: Exiting with correct code [S.UUUs] +expect_all_pass:failure -> expected_failure: Oh no: Forced failure [S.UUUs] +expect_all_pass:signal -> expected_failure: Exiting with correct signal [S.UUUs] +expect_all_pass:timeout -> expected_failure: This times out [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +5/5 passed (0 failed) +EOF +# CHECK_STYLE_ENABLE + + utils_cp_helper expect_all_pass . + atf_check -s exit:0 -o file:expout -e empty kyua test +} + + +utils_test_case expect__some_fail +expect__some_fail_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="expect_some_fail"} +EOF + +# CHECK_STYLE_DISABLE + cat >expout <<EOF +expect_some_fail:die -> failed: Test case was expected to terminate abruptly but it continued execution [S.UUUs] +expect_some_fail:exit -> failed: Test case expected to exit with code 12 but got code 34 [S.UUUs] +expect_some_fail:failure -> failed: Test case was expecting a failure but none were raised [S.UUUs] +expect_some_fail:pass -> passed [S.UUUs] +expect_some_fail:signal -> failed: Test case expected to receive signal 15 but got 9 [S.UUUs] +expect_some_fail:timeout -> failed: Test case was expected to hang but it continued execution [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/6 passed (5 failed) +EOF +# CHECK_STYLE_ENABLE + + utils_cp_helper expect_some_fail . + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case premature_exit +premature_exit_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="bogus_test_cases"} +EOF + +# CHECK_STYLE_DISABLE + cat >expout <<EOF +bogus_test_cases:die -> broken: Premature exit; test case received signal 9 [S.UUUs] +bogus_test_cases:exit -> broken: Premature exit; test case exited with code 0 [S.UUUs] +bogus_test_cases:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/3 passed (2 failed) +EOF +# CHECK_STYLE_ENABLE + + utils_cp_helper bogus_test_cases . + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case no_args +no_args_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="simple_all_pass"} +include("subdir/Kyuafile") +EOF + utils_cp_helper metadata . + utils_cp_helper simple_all_pass . + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration2") +atf_test_program{name="simple_some_fail"} +EOF + utils_cp_helper simple_some_fail subdir + + cat >expout <<EOF +simple_all_pass:pass -> passed [S.UUUs] +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/simple_some_fail:fail -> failed: This fails on purpose [S.UUUs] +subdir/simple_some_fail:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +3/4 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case one_arg__subdir +one_arg__subdir_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +include("subdir/Kyuafile") +EOF + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("in-subdir") +atf_test_program{name="simple_all_pass"} +EOF + utils_cp_helper simple_all_pass subdir + +# CHECK_STYLE_DISABLE + cat >expout <<EOF +subdir/simple_all_pass:pass -> passed [S.UUUs] +subdir/simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF +# CHECK_STYLE_ENABLE + atf_check -s exit:0 -o file:expout -e empty kyua test subdir +} + + +utils_test_case one_arg__test_case +one_arg__test_case_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + + cat >expout <<EOF +first:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/1 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test first:skip +} + + +utils_test_case one_arg__test_program +one_arg__test_program_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_some_fail second + + cat >expout <<EOF +second:fail -> failed: This fails on purpose [S.UUUs] +second:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/2 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test second +} + + +utils_test_case one_arg__invalid +one_arg__invalid_body() { +cat >experr <<EOF +kyua: E: Test case component in 'foo:' is empty. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua test foo: + +cat >experr <<EOF +kyua: E: Program name '/a/b' must be relative to the test suite, not absolute. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua test /a/b +} + + +utils_test_case many_args__ok +many_args__ok_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +include("subdir/Kyuafile") +atf_test_program{name="first"} +EOF + utils_cp_helper simple_all_pass first + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("in-subdir") +atf_test_program{name="second"} +EOF + utils_cp_helper simple_some_fail subdir/second + + cat >expout <<EOF +first:pass -> passed [S.UUUs] +subdir/second:fail -> failed: This fails on purpose [S.UUUs] +subdir/second:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/3 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test subdir first:pass +} + + +utils_test_case many_args__invalid +many_args__invalid_body() { +cat >experr <<EOF +kyua: E: Program name component in ':badbad' is empty. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua test this-is-ok :badbad + +cat >experr <<EOF +kyua: E: Program name '/foo' must be relative to the test suite, not absolute. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua test this-is-ok /foo +} + + +utils_test_case many_args__no_match__all +many_args__no_match__all_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + + cat >expout <<EOF +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) +EOF + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'first1'. +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua test first1 +} + + +utils_test_case many_args__no_match__some +many_args__no_match__some_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("top-level") +atf_test_program{name="first"} +atf_test_program{name="second"} +atf_test_program{name="third"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + utils_cp_helper simple_some_fail third + + cat >expout <<EOF +first:pass -> passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +third:fail -> failed: This fails on purpose [S.UUUs] +third:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +3/4 passed (1 failed) +EOF + + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'fifth'. +kyua: W: No test cases matched by the filter 'fourth'. +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua test first fourth \ + third fifth +} + + +utils_test_case args_are_relative +args_are_relative_body() { + utils_install_stable_test_wrapper + + mkdir root + cat >root/Kyuafile <<EOF +syntax(2) +test_suite("integration-1") +atf_test_program{name="first"} +atf_test_program{name="second"} +include("subdir/Kyuafile") +EOF + utils_cp_helper simple_all_pass root/first + utils_cp_helper simple_some_fail root/second + + mkdir root/subdir + cat >root/subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration-2") +atf_test_program{name="third"} +atf_test_program{name="fourth"} +EOF + utils_cp_helper simple_all_pass root/subdir/third + utils_cp_helper simple_some_fail root/subdir/fourth + + cat >expout <<EOF +first:pass -> passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/fourth:fail -> failed: This fails on purpose [S.UUUs] + +Results file id is $(utils_results_id root) +Results saved to $(utils_results_file root) + +2/3 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test \ + -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail +} + + +utils_test_case only_load_used_test_programs +only_load_used_test_programs_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +atf_test_program{name="second"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper bad_test_program second + + cat >expout <<EOF +first:pass -> passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE + atf_check -s exit:0 -o file:expout -e empty kyua test first + if [ -f "${CREATE_COOKIE}" ]; then + atf_fail "An unmatched test case has been executed, which harms" \ + "performance" + fi +} + + +utils_test_case config_behavior +config_behavior_body() { + cat >"my-config" <<EOF +syntax(2) +test_suites.suite1["the-variable"] = "value1" +test_suites.suite2["the-variable"] = "override me" +EOF + + cat >Kyuafile <<EOF +syntax(2) +atf_test_program{name="config1", test_suite="suite1"} +atf_test_program{name="config2", test_suite="suite2"} +atf_test_program{name="config3", test_suite="suite3"} +EOF + utils_cp_helper config config1 + utils_cp_helper config config2 + utils_cp_helper config config3 + + atf_check -s exit:1 -o save:stdout -e empty \ + kyua -c my-config -v test_suites.suite2.the-variable=value2 test + atf_check -s exit:0 -o ignore -e empty \ + grep 'config1:get_variable.*failed' stdout + atf_check -s exit:0 -o ignore -e empty \ + grep 'config2:get_variable.*passed' stdout + atf_check -s exit:0 -o ignore -e empty \ + grep 'config3:get_variable.*skipped' stdout + + CONFIG_VAR_FILE="$(pwd)/cookie"; export CONFIG_VAR_FILE + if [ -f "${CONFIG_VAR_FILE}" ]; then + atf_fail "Cookie file already created; test case list may have gotten" \ + "a bad configuration" + fi + atf_check -s exit:1 -o ignore -e empty kyua -c my-config test config1 + [ -f "${CONFIG_VAR_FILE}" ] || \ + atf_fail "Cookie file not created; test case list did not get" \ + "configuration variables" + value="$(cat "${CONFIG_VAR_FILE}")" + [ "${value}" = "value1" ] || \ + atf_fail "Invalid value (${value}) in cookie file; test case list did" \ + "not get the correct configuration variables" +} + + +utils_test_case store_contents +store_contents_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +atf_test_program{name="some-program", test_suite="suite1"} +EOF + utils_cp_helper simple_some_fail some-program + cat >expout <<EOF +some-program:fail -> failed: This fails on purpose [S.UUUs] +some-program:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/2 passed (1 failed) +EOF + + atf_check -s exit:1 -o file:expout -e empty kyua test + +cat >expout <<EOF +some-program,fail,failed,This fails on purpose +some-program,pass,passed,NULL +EOF + atf_check -s exit:0 -o file:expout -e empty \ + kyua db-exec --no-headers \ + "SELECT " \ + " test_programs.relative_path, test_cases.name, " \ + " test_results.result_type, test_results.result_reason " \ + "FROM test_programs " \ + " JOIN test_cases " \ + " ON test_programs.test_program_id = test_cases.test_program_id " \ + " JOIN test_results " \ + " ON test_cases.test_case_id = test_results.test_case_id " \ + "ORDER BY test_programs.relative_path, test_cases.name" +} + + +utils_test_case results_file__ok +results_file__ok_body() { + cat >Kyuafile <<EOF +syntax(2) +atf_test_program{name="config1", test_suite="suite1"} +EOF + utils_cp_helper config config1 + + atf_check -s exit:0 -o ignore -e empty kyua test -r foo1.db + test -f foo1.db || atf_fail "-s did not work" + atf_check -s exit:0 -o ignore -e empty kyua test --results-file=foo2.db + test -f foo2.db || atf_fail "--results-file did not work" + test ! -f .kyua/store.db || atf_fail "Default database created" +} + + +utils_test_case results_file__fail +results_file__fail_body() { + cat >Kyuafile <<EOF +syntax(2) +atf_test_program{name="config1", test_suite="suite1"} +EOF + utils_cp_helper config config1 + + atf_check -s exit:3 -o empty -e match:"Invalid.*--results-file" \ + kyua test --results-file= +} + + +utils_test_case results_file__reuse +results_file__reuse_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +atf_test_program{name="simple_all_pass", test_suite="integration"} +EOF + utils_cp_helper simple_all_pass . + atf_check -s exit:0 -o ignore -e empty kyua test -r results.db + + atf_check -s exit:2 -o empty -e match:"results.db already exists" \ + kyua test --results-file="results.db" +} + + +utils_test_case build_root_flag +build_root_flag_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +include("subdir/Kyuafile") +EOF + + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="second"} +atf_test_program{name="third"} +EOF + + cat >expout <<EOF +first:pass -> passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/second:pass -> passed [S.UUUs] +subdir/second:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/third:pass -> passed [S.UUUs] +subdir/third:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +6/6 passed (0 failed) +EOF + + mkdir build + mkdir build/subdir + utils_cp_helper simple_all_pass build/first + utils_cp_helper simple_all_pass build/subdir/second + utils_cp_helper simple_all_pass build/subdir/third + + atf_check -s exit:0 -o file:expout -e empty kyua test --build-root=build +} + + +utils_test_case kyuafile_flag__no_args +kyuafile_flag__no_args_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +This file is bogus but it is not loaded. +EOF + + cat >myfile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="sometest"} +EOF + utils_cp_helper simple_all_pass sometest + + cat >expout <<EOF +sometest:pass -> passed [S.UUUs] +sometest:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile + atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile +} + + +utils_test_case kyuafile_flag__some_args +kyuafile_flag__some_args_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +This file is bogus but it is not loaded. +EOF + + cat >myfile <<EOF +syntax(2) +test_suite("hello-world") +atf_test_program{name="sometest"} +EOF + utils_cp_helper simple_all_pass sometest + + cat >expout <<EOF +sometest:pass -> passed [S.UUUs] +sometest:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile sometest + cat >expout <<EOF +sometest:pass -> passed [S.UUUs] +sometest:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile \ + sometest +} + + +utils_test_case interrupt +interrupt_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="interrupts"} +EOF + utils_cp_helper interrupts . + + kyua \ + -v test_suites.integration.body-cookie="$(pwd)/body" \ + -v test_suites.integration.cleanup-cookie="$(pwd)/cleanup" \ + test >stdout 2>stderr & + pid=${!} + echo "Kyua subprocess is PID ${pid}" + + while [ ! -f body ]; do + echo "Waiting for body to start" + sleep 1 + done + echo "Body started" + sleep 1 + + echo "Sending INT signal to ${pid}" + kill -INT ${pid} + echo "Waiting for process ${pid} to exit" + wait ${pid} + ret=${?} + sed -e 's,^,kyua stdout:,' stdout + sed -e 's,^,kyua stderr:,' stderr + echo "Process ${pid} exited" + [ ${ret} -ne 0 ] || atf_fail 'No error code reported' + + [ -f cleanup ] || atf_fail 'Cleanup part not executed after signal' + atf_expect_pass + + atf_check -s exit:0 -o ignore -e empty grep 'Signal caught' stderr + atf_check -s exit:0 -o ignore -e empty \ + grep 'kyua: E: Interrupted by signal' stderr +} + + +utils_test_case exclusive_tests +exclusive_tests_body() { + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +EOF + for i in $(seq 100); do + echo 'plain_test_program{name="race", is_exclusive=true}' >>Kyuafile + done + utils_cp_helper race . + + atf_check \ + -s exit:0 \ + -o match:"100/100 passed" \ + kyua \ + -v parallelism=20 \ + -v test_suites.integration.shared_file="$(pwd)/shared_file" \ + test +} + + +utils_test_case no_test_program_match +no_test_program_match_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +EOF + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + + cat >expout <<EOF +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) +EOF + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'second'. +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua test second +} + + +utils_test_case no_test_case_match +no_test_case_match_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="first"} +EOF + utils_cp_helper simple_all_pass first + + cat >expout <<EOF +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) +EOF + cat >experr <<EOF +kyua: W: No test cases matched by the filter 'first:foobar'. +EOF + atf_check -s exit:1 -o file:expout -e file:experr kyua test first:foobar +} + + +utils_test_case missing_kyuafile__no_args +missing_kyuafile__no_args_body() { + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua test +} + + +utils_test_case missing_kyuafile__test_program +missing_kyuafile__test_program_body() { + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="unused"} +EOF + utils_cp_helper simple_all_pass subdir/unused + + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua test subdir/unused +} + + +utils_test_case missing_kyuafile__subdir +missing_kyuafile__subdir_body() { + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="unused"} +EOF + utils_cp_helper simple_all_pass subdir/unused + + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. +EOF + atf_check -s exit:2 -o empty -e file:experr kyua test subdir +} + + +utils_test_case bogus_config +bogus_config_body() { + mkdir .kyua + cat >"${HOME}/.kyua/kyua.conf" <<EOF +Hello, world. +EOF + + file_re='.*\.kyua/kyua.conf' + atf_check -s exit:2 -o empty \ + -e match:"^kyua: E: Load of '${file_re}' failed: Failed to load Lua" \ + kyua test +} + + +utils_test_case bogus_kyuafile +bogus_kyuafile_body() { + cat >Kyuafile <<EOF +Hello, world. +EOF + atf_check -s exit:2 -o empty \ + -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list +} + + +utils_test_case bogus_test_program +bogus_test_program_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="crash_on_list"} +atf_test_program{name="non_executable"} +EOF + utils_cp_helper bad_test_program crash_on_list + echo 'I am not executable' >non_executable + +# CHECK_STYLE_DISABLE + cat >expout <<EOF +crash_on_list:__test_cases_list__ -> broken: Invalid header for test case list; expecting Content-Type for application/X-atf-tp version 1, got '' [S.UUUs] +non_executable:__test_cases_list__ -> broken: Permission denied to run test program [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +0/2 passed (2 failed) +EOF +# CHECK_STYLE_ENABLE + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case missing_test_program +missing_test_program_body() { + cat >Kyuafile <<EOF +syntax(2) +include("subdir/Kyuafile") +EOF + mkdir subdir + cat >subdir/Kyuafile <<EOF +syntax(2) +test_suite("integration") +atf_test_program{name="ok"} +atf_test_program{name="i-am-missing"} +EOF + echo 'I should not be touched because the Kyuafile is bogus' >subdir/ok + +# CHECK_STYLE_DISABLE + cat >experr <<EOF +kyua: E: Load of 'Kyuafile' failed: .*Non-existent test program 'subdir/i-am-missing'. +EOF +# CHECK_STYLE_ENABLE + atf_check -s exit:2 -o empty -e "match:$(cat experr)" kyua list +} + + +atf_init_test_cases() { + atf_add_test_case one_test_program__all_pass + atf_add_test_case one_test_program__some_fail + atf_add_test_case many_test_programs__all_pass + atf_add_test_case many_test_programs__some_fail + atf_add_test_case expect__all_pass + atf_add_test_case expect__some_fail + atf_add_test_case premature_exit + + atf_add_test_case no_args + atf_add_test_case one_arg__subdir + atf_add_test_case one_arg__test_case + atf_add_test_case one_arg__test_program + atf_add_test_case one_arg__invalid + atf_add_test_case many_args__ok + atf_add_test_case many_args__invalid + atf_add_test_case many_args__no_match__all + atf_add_test_case many_args__no_match__some + + atf_add_test_case args_are_relative + + atf_add_test_case only_load_used_test_programs + + atf_add_test_case config_behavior + + atf_add_test_case store_contents + atf_add_test_case results_file__ok + atf_add_test_case results_file__fail + atf_add_test_case results_file__reuse + + atf_add_test_case build_root_flag + + atf_add_test_case kyuafile_flag__no_args + atf_add_test_case kyuafile_flag__some_args + + atf_add_test_case interrupt + + atf_add_test_case exclusive_tests + + atf_add_test_case no_test_program_match + atf_add_test_case no_test_case_match + + atf_add_test_case missing_kyuafile__no_args + atf_add_test_case missing_kyuafile__test_program + atf_add_test_case missing_kyuafile__subdir + + atf_add_test_case bogus_config + atf_add_test_case bogus_kyuafile + atf_add_test_case bogus_test_program + atf_add_test_case missing_test_program +} diff --git a/integration/global_test.sh b/integration/global_test.sh new file mode 100755 index 000000000000..9cc8be2d1dec --- /dev/null +++ b/integration/global_test.sh @@ -0,0 +1,146 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case no_args +no_args_body() { + cat >experr <<EOF +Usage error: No command provided. +Type 'kyua help' for usage information. +EOF + + atf_check -s exit:3 -o empty -e file:experr kyua +} + + +utils_test_case unknown_option +unknown_option_body() { + cat >experr <<EOF +Usage error: Unknown option --this_is_unknown. +Type 'kyua help' for usage information. +EOF + + atf_check -s exit:3 -o empty -e file:experr kyua --this_is_unknown +} + + +utils_test_case unknown_command +unknown_command_body() { + cat >experr <<EOF +Usage error: Unknown command 'i_am_not_known'. +Type 'kyua help' for usage information. +EOF + + atf_check -s exit:3 -o empty -e file:experr kyua i_am_not_known +} + + +utils_test_case logfile__default +logfile__default_body() { + atf_check -s exit:0 test ! -d .kyua/logs/ + atf_check -s exit:3 -o empty -e ignore kyua + atf_check -s exit:0 test -d .kyua/logs/ +} + + +utils_test_case logfile__override +logfile__override_body() { + atf_check -s exit:0 test ! -f test.log + atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log + + atf_check -s exit:0 test ! -d .kyua/logs/ + atf_check -s exit:0 test -f test.log + + grep ' E .* No command provided' test.log || atf_fail "Log file does" \ + "contain required message" +} + + +utils_test_case loglevel__default +loglevel__default_body() { + atf_check -s exit:0 test ! -f test.log + atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log + + atf_check -s exit:0 test ! -d .kyua/logs/ + atf_check -s exit:0 test -f test.log + + grep ' E .* No command provided' test.log || atf_fail "Log file does" \ + "contain required message" + if grep ' D ' test.log; then + atf_fail "Log file contains debug messages but it should not" + fi +} + + +utils_test_case loglevel__lower +loglevel__lower_body() { + atf_check -s exit:0 test ! -f test.log + atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log \ + --loglevel=warning + + atf_check -s exit:0 test ! -d .kyua/logs/ + atf_check -s exit:0 test -f test.log + + grep ' E .* No command provided' test.log || atf_fail "Log file does" \ + "contain required message" + if grep ' I ' test.log; then + atf_fail "Log file contains info messages but it should not" + fi +} + + +utils_test_case loglevel__higher +loglevel__higher_body() { + atf_check -s exit:0 test ! -f test.log + atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log \ + --loglevel=debug + + atf_check -s exit:0 test ! -d .kyua/logs/ + atf_check -s exit:0 test -f test.log + + grep ' E .* No command provided' test.log || atf_fail "Log file does" \ + "contain required message" + grep ' D ' test.log || atf_fail "Log file does not contain debug messages" +} + + +atf_init_test_cases() { + atf_add_test_case no_args + atf_add_test_case unknown_option + atf_add_test_case unknown_command + + atf_add_test_case logfile__default + atf_add_test_case logfile__override + + atf_add_test_case loglevel__default + atf_add_test_case loglevel__lower + atf_add_test_case loglevel__higher + + # Tests for the global configuration-related flags are found in the + # cmd_config_test test program. +} diff --git a/integration/helpers/.gitignore b/integration/helpers/.gitignore new file mode 100644 index 000000000000..11634bcfeb85 --- /dev/null +++ b/integration/helpers/.gitignore @@ -0,0 +1,11 @@ +bad_test_program +bogus_test_cases +config +dump_env +expect_all_pass +expect_some_fail +interrupts +metadata +race +simple_all_pass +simple_some_fail diff --git a/integration/helpers/Makefile.am.inc b/integration/helpers/Makefile.am.inc new file mode 100644 index 000000000000..d835aba51c67 --- /dev/null +++ b/integration/helpers/Makefile.am.inc @@ -0,0 +1,90 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if WITH_ATF +tests_integration_helpersdir = $(pkgtestsdir)/integration/helpers + +tests_integration_helpers_PROGRAMS = integration/helpers/bad_test_program +integration_helpers_bad_test_program_SOURCES = \ + integration/helpers/bad_test_program.cpp + +tests_integration_helpers_PROGRAMS += integration/helpers/bogus_test_cases +integration_helpers_bogus_test_cases_SOURCES = \ + integration/helpers/bogus_test_cases.cpp +integration_helpers_bogus_test_cases_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_bogus_test_cases_LDADD = $(ATF_CXX_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/config +integration_helpers_config_SOURCES = integration/helpers/config.cpp +integration_helpers_config_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_config_LDADD = $(ATF_CXX_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/dump_env +integration_helpers_dump_env_SOURCES = integration/helpers/dump_env.cpp +integration_helpers_dump_env_CXXFLAGS = $(UTILS_CFLAGS) +integration_helpers_dump_env_LDADD = $(UTILS_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/expect_all_pass +integration_helpers_expect_all_pass_SOURCES = \ + integration/helpers/expect_all_pass.cpp +integration_helpers_expect_all_pass_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_expect_all_pass_LDADD = $(ATF_CXX_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/expect_some_fail +integration_helpers_expect_some_fail_SOURCES = \ + integration/helpers/expect_some_fail.cpp +integration_helpers_expect_some_fail_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_expect_some_fail_LDADD = $(ATF_CXX_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/interrupts +integration_helpers_interrupts_SOURCES = integration/helpers/interrupts.cpp +integration_helpers_interrupts_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_interrupts_LDADD = $(ATF_CXX_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/metadata +integration_helpers_metadata_SOURCES = integration/helpers/metadata.cpp +integration_helpers_metadata_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_metadata_LDADD = $(ATF_CXX_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/race +integration_helpers_race_SOURCES = integration/helpers/race.cpp +integration_helpers_race_CXXFLAGS = $(UTILS_CFLAGS) +integration_helpers_race_LDADD = $(UTILS_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/simple_all_pass +integration_helpers_simple_all_pass_SOURCES = \ + integration/helpers/simple_all_pass.cpp +integration_helpers_simple_all_pass_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_simple_all_pass_LDADD = $(ATF_CXX_LIBS) + +tests_integration_helpers_PROGRAMS += integration/helpers/simple_some_fail +integration_helpers_simple_some_fail_SOURCES = \ + integration/helpers/simple_some_fail.cpp +integration_helpers_simple_some_fail_CXXFLAGS = $(ATF_CXX_CFLAGS) +integration_helpers_simple_some_fail_LDADD = $(ATF_CXX_LIBS) +endif diff --git a/integration/helpers/bad_test_program.cpp b/integration/helpers/bad_test_program.cpp new file mode 100644 index 000000000000..210709ee8976 --- /dev/null +++ b/integration/helpers/bad_test_program.cpp @@ -0,0 +1,50 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <cstdlib> +#include <cstring> +#include <fstream> +#include <iostream> + + +int +main(void) +{ + std::cerr << "This is not a valid test program!\n"; + + const char* cookie = std::getenv("CREATE_COOKIE"); + if (cookie != NULL && std::strlen(cookie) > 0) { + std::ofstream file(cookie); + if (!file) + std::abort(); + file << "Cookie file\n"; + file.close(); + } + + return EXIT_SUCCESS; +} diff --git a/integration/helpers/bogus_test_cases.cpp b/integration/helpers/bogus_test_cases.cpp new file mode 100644 index 000000000000..1a7c27031e1b --- /dev/null +++ b/integration/helpers/bogus_test_cases.cpp @@ -0,0 +1,64 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <signal.h> +#include <unistd.h> +} + +#include <cstdlib> + +#include <atf-c++.hpp> + + +ATF_TEST_CASE_WITHOUT_HEAD(die); +ATF_TEST_CASE_BODY(die) +{ + ::kill(::getpid(), SIGKILL); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(exit); +ATF_TEST_CASE_BODY(exit) +{ + std::exit(EXIT_SUCCESS); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, die); + ATF_ADD_TEST_CASE(tcs, exit); + ATF_ADD_TEST_CASE(tcs, pass); +} diff --git a/integration/helpers/config.cpp b/integration/helpers/config.cpp new file mode 100644 index 000000000000..aa4fef291725 --- /dev/null +++ b/integration/helpers/config.cpp @@ -0,0 +1,58 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <cstdlib> + +#include <atf-c++.hpp> + + +ATF_TEST_CASE(get_variable); +ATF_TEST_CASE_HEAD(get_variable) +{ + const char* output = ::getenv("CONFIG_VAR_FILE"); + if (output == NULL) { + set_md_var("require.config", "the-variable"); + } else { + if (has_config_var("the-variable")) { + atf::utils::create_file(output, get_config_var("the-variable") + + std::string("\n")); + } else { + atf::utils::create_file(output, "NOT DEFINED\n"); + } + } +} +ATF_TEST_CASE_BODY(get_variable) +{ + ATF_REQUIRE_EQ("value2", get_config_var("the-variable")); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, get_variable); +} diff --git a/integration/helpers/dump_env.cpp b/integration/helpers/dump_env.cpp new file mode 100644 index 000000000000..a2e8313a0062 --- /dev/null +++ b/integration/helpers/dump_env.cpp @@ -0,0 +1,74 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Dumps all environment variables. +// +// This helper program allows comparing the printed environment variables +// to what 'kyua report --verbose' may output. It does so by sorting the +// variables and allowing the caller to customize how the output looks +// like (indentation for each line and for continuation lines). + +#include <cstdlib> +#include <iostream> + +#include "utils/env.hpp" +#include "utils/text/operations.ipp" + +namespace text = utils::text; + + +int +main(const int argc, const char* const* const argv) +{ + if (argc != 3) { + std::cerr << "Usage: dump_env <prefix> <continuation-prefix>\n"; + return EXIT_FAILURE; + } + const char* prefix = argv[1]; + const char* continuation_prefix = argv[2]; + + const std::map< std::string, std::string > env = utils::getallenv(); + for (std::map< std::string, std::string >::const_iterator + iter = env.begin(); iter != env.end(); ++iter) { + const std::string& name = (*iter).first; + const std::vector< std::string > value = text::split( + (*iter).second, '\n'); + + if (value.empty()) { + std::cout << prefix << name << "=\n"; + } else { + std::cout << prefix << name << '=' << value[0] << '\n'; + for (std::vector< std::string >::size_type i = 1; + i < value.size(); ++i) { + std::cout << continuation_prefix << value[i] << '\n'; + } + } + } + + return EXIT_SUCCESS; +} diff --git a/integration/helpers/expect_all_pass.cpp b/integration/helpers/expect_all_pass.cpp new file mode 100644 index 000000000000..a7df16e3a783 --- /dev/null +++ b/integration/helpers/expect_all_pass.cpp @@ -0,0 +1,92 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <signal.h> +#include <unistd.h> +} + +#include <cstdlib> + +#include <atf-c++.hpp> + +#include "utils/test_utils.ipp" + + +ATF_TEST_CASE_WITHOUT_HEAD(die); +ATF_TEST_CASE_BODY(die) +{ + expect_death("This is the reason for death"); + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(exit); +ATF_TEST_CASE_BODY(exit) +{ + expect_exit(12, "Exiting with correct code"); + std::exit(12); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(failure); +ATF_TEST_CASE_BODY(failure) +{ + expect_fail("Oh no"); + fail("Forced failure"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(signal); +ATF_TEST_CASE_BODY(signal) +{ + expect_signal(SIGTERM, "Exiting with correct signal"); + ::kill(::getpid(), SIGTERM); +} + + +ATF_TEST_CASE(timeout); +ATF_TEST_CASE_HEAD(timeout) +{ + set_md_var("timeout", "1"); +} +ATF_TEST_CASE_BODY(timeout) +{ + expect_timeout("This times out"); + ::sleep(10); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, die); + ATF_ADD_TEST_CASE(tcs, exit); + ATF_ADD_TEST_CASE(tcs, failure); + ATF_ADD_TEST_CASE(tcs, signal); + ATF_ADD_TEST_CASE(tcs, timeout); +} diff --git a/integration/helpers/expect_some_fail.cpp b/integration/helpers/expect_some_fail.cpp new file mode 100644 index 000000000000..da1a9f0ceb39 --- /dev/null +++ b/integration/helpers/expect_some_fail.cpp @@ -0,0 +1,94 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <signal.h> +#include <unistd.h> +} + +#include <cstdlib> + +#include <atf-c++.hpp> + + +ATF_TEST_CASE_WITHOUT_HEAD(die); +ATF_TEST_CASE_BODY(die) +{ + expect_death("Won't die"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(exit); +ATF_TEST_CASE_BODY(exit) +{ + expect_exit(12, "Invalid exit code"); + std::exit(34); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(failure); +ATF_TEST_CASE_BODY(failure) +{ + expect_fail("Does not fail"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ +} + + +ATF_TEST_CASE_WITHOUT_HEAD(signal); +ATF_TEST_CASE_BODY(signal) +{ + expect_signal(SIGTERM, "Invalid signal"); + ::kill(::getpid(), SIGKILL); +} + + +ATF_TEST_CASE(timeout); +ATF_TEST_CASE_HEAD(timeout) +{ + set_md_var("timeout", "1"); +} +ATF_TEST_CASE_BODY(timeout) +{ + expect_timeout("Does not time out"); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, die); + ATF_ADD_TEST_CASE(tcs, exit); + ATF_ADD_TEST_CASE(tcs, failure); + ATF_ADD_TEST_CASE(tcs, pass); + ATF_ADD_TEST_CASE(tcs, signal); + ATF_ADD_TEST_CASE(tcs, timeout); +} diff --git a/integration/helpers/interrupts.cpp b/integration/helpers/interrupts.cpp new file mode 100644 index 000000000000..b6c5a948098c --- /dev/null +++ b/integration/helpers/interrupts.cpp @@ -0,0 +1,62 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <unistd.h> +} + +#include <atf-c++.hpp> + +#include <fstream> + + +ATF_TEST_CASE_WITH_CLEANUP(block_body); +ATF_TEST_CASE_HEAD(block_body) +{ + set_md_var("require.config", "body-cookie cleanup-cookie"); +} +ATF_TEST_CASE_BODY(block_body) +{ + const std::string cookie(get_config_var("body-cookie")); + std::ofstream output(cookie.c_str()); + output.close(); + for (;;) + ::pause(); +} +ATF_TEST_CASE_CLEANUP(block_body) +{ + const std::string cookie(get_config_var("cleanup-cookie")); + std::ofstream output(cookie.c_str()); + output.close(); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, block_body); +} diff --git a/integration/helpers/metadata.cpp b/integration/helpers/metadata.cpp new file mode 100644 index 000000000000..8005d7d9b68d --- /dev/null +++ b/integration/helpers/metadata.cpp @@ -0,0 +1,95 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <cstdlib> +#include <iostream> + +#include <atf-c++.hpp> + +#include "utils/test_utils.ipp" + + +ATF_TEST_CASE_WITHOUT_HEAD(no_properties); +ATF_TEST_CASE_BODY(no_properties) +{ +} + + +ATF_TEST_CASE(one_property); +ATF_TEST_CASE_HEAD(one_property) +{ + set_md_var("descr", "Does nothing but has one metadata property"); +} +ATF_TEST_CASE_BODY(one_property) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE(many_properties); +ATF_TEST_CASE_HEAD(many_properties) +{ + set_md_var("descr", " A description with some padding"); + set_md_var("require.arch", "some-architecture"); + set_md_var("require.config", "var1 var2 var3"); + set_md_var("require.files", "/my/file1 /some/other/file"); + set_md_var("require.machine", "some-platform"); + set_md_var("require.progs", "bin1 bin2 /nonexistent/bin3"); + set_md_var("require.user", "root"); + set_md_var("X-no-meaning", "I am a custom variable"); +} +ATF_TEST_CASE_BODY(many_properties) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE_WITH_CLEANUP(with_cleanup); +ATF_TEST_CASE_HEAD(with_cleanup) +{ + set_md_var("timeout", "250"); +} +ATF_TEST_CASE_BODY(with_cleanup) +{ + std::cout << "Body message to stdout\n"; + std::cerr << "Body message to stderr\n"; +} +ATF_TEST_CASE_CLEANUP(with_cleanup) +{ + std::cout << "Cleanup message to stdout\n"; + std::cerr << "Cleanup message to stderr\n"; +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, no_properties); + ATF_ADD_TEST_CASE(tcs, one_property); + ATF_ADD_TEST_CASE(tcs, many_properties); + ATF_ADD_TEST_CASE(tcs, with_cleanup); +} diff --git a/integration/helpers/race.cpp b/integration/helpers/race.cpp new file mode 100644 index 000000000000..39d4b04f3923 --- /dev/null +++ b/integration/helpers/race.cpp @@ -0,0 +1,99 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file integration/helpers/race.cpp +/// Creates a file and reads it back, looking for races. +/// +/// This program should fail with high chances if it is called multiple times at +/// once with TEST_ENV_shared_file pointing to the same file. + +extern "C" { +#include <sys/types.h> + +#include <unistd.h> +} + +#include <cstdlib> +#include <fstream> +#include <iostream> + +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/env.hpp" +#include "utils/optional.ipp" +#include "utils/stream.hpp" + +namespace fs = utils::fs; + +using utils::optional; + + +/// Entry point to the helper test program. +/// +/// \return EXIT_SUCCESS if no race is detected; EXIT_FAILURE otherwise. +int +main(void) +{ + const optional< std::string > shared_file = utils::getenv( + "TEST_ENV_shared_file"); + if (!shared_file) { + std::cerr << "Environment variable TEST_ENV_shared_file not defined\n"; + std::exit(EXIT_FAILURE); + } + const fs::path shared_path(shared_file.get()); + + if (fs::exists(shared_path)) { + std::cerr << "Shared file already exists; created by a concurrent " + "test?"; + std::exit(EXIT_FAILURE); + } + + const std::string contents = F("%s") % ::getpid(); + + std::ofstream output(shared_path.c_str()); + if (!output) { + std::cerr << "Failed to create shared file; conflict with a concurrent " + "test?"; + std::exit(EXIT_FAILURE); + } + output << contents; + output.close(); + + ::usleep(10000); + + const std::string read_contents = utils::read_file(shared_path); + if (read_contents != contents) { + std::cerr << "Shared file contains unexpected contents; modified by a " + "concurrent test?"; + std::exit(EXIT_FAILURE); + } + + fs::unlink(shared_path); + std::exit(EXIT_SUCCESS); +} diff --git a/integration/helpers/simple_all_pass.cpp b/integration/helpers/simple_all_pass.cpp new file mode 100644 index 000000000000..4e168b4cca5f --- /dev/null +++ b/integration/helpers/simple_all_pass.cpp @@ -0,0 +1,55 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <iostream> + +#include <atf-c++.hpp> + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ + std::cout << "This is the stdout of pass\n"; + std::cerr << "This is the stderr of pass\n"; +} + + +ATF_TEST_CASE_WITHOUT_HEAD(skip); +ATF_TEST_CASE_BODY(skip) +{ + std::cout << "This is the stdout of skip\n"; + std::cerr << "This is the stderr of skip\n"; + skip("The reason for skipping is this"); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, pass); + ATF_ADD_TEST_CASE(tcs, skip); +} diff --git a/integration/helpers/simple_some_fail.cpp b/integration/helpers/simple_some_fail.cpp new file mode 100644 index 000000000000..909ffb6e2ee1 --- /dev/null +++ b/integration/helpers/simple_some_fail.cpp @@ -0,0 +1,53 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <iostream> + +#include <atf-c++.hpp> + + +ATF_TEST_CASE_WITHOUT_HEAD(fail); +ATF_TEST_CASE_BODY(fail) +{ + std::cout << "This is the stdout of fail\n"; + std::cerr << "This is the stderr of fail\n"; + fail("This fails on purpose"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, fail); + ATF_ADD_TEST_CASE(tcs, pass); +} diff --git a/integration/utils.sh b/integration/utils.sh new file mode 100755 index 000000000000..99565a1c9857 --- /dev/null +++ b/integration/utils.sh @@ -0,0 +1,177 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Subcommand to strip out the durations and timestamps in a report. +# +# This is to make the reports deterministic and thus easily testable. The +# time deltas are replaced by the fixed string S.UUU and the timestamps are +# replaced by the fixed strings YYYYMMDD.HHMMSS.ssssss and +# YYYY-MM-DDTHH:MM:SS.ssssssZ depending on their original format. +# +# This variable should be used as shown here: +# +# atf_check ... -x kyua report "| ${utils_strip_times}" +# +# Use the utils_install_times_wrapper function to create a 'kyua' wrapper +# script that automatically does this. +# CHECK_STYLE_DISABLE +utils_strip_times='sed -E \ + -e "s,( |\[|\")[0-9][0-9]*.[0-9][0-9][0-9](s]|s|\"),\1S.UUU\2,g" \ + -e "s,[0-9]{8}-[0-9]{6}-[0-9]{6},YYYYMMDD-HHMMSS-ssssss,g" \ + -e "s,[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z,YYYY-MM-DDTHH:MM:SS.ssssssZ,g"' +# CHECK_STYLE_ENABLE + + +# Same as utils_strip_times but avoids stripping timestamp-based report IDs. +# +# This is to make the reports deterministic and thus easily testable. The +# time deltas are replaced by the fixed string S.UUU and the timestamps are +# replaced by the fixed string YYYY-MM-DDTHH:MM:SS.ssssssZ. +# CHECK_STYLE_DISABLE +utils_strip_times_but_not_ids='sed -E \ + -e "s,( |\[|\")[0-9][0-9]*.[0-9][0-9][0-9](s]|s|\"),\1S.UUU\2,g" \ + -e "s,[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z,YYYY-MM-DDTHH:MM:SS.ssssssZ,g"' +# CHECK_STYLE_ENABLE + + +# Computes the results id for a test suite run. +# +# The computed path is "generic" in the sense that it does not include a +# real timestamp: it only includes a placeholder. This function should be +# used along the utils_strip_times function so that the timestamps of +# the real results files are stripped out. +# +# \param path Optional path to use; if not given, use the cwd. +utils_results_id() { + local test_suite_id="$(utils_test_suite_id "${@}")" + echo "${test_suite_id}.YYYYMMDD-HHMMSS-ssssss" +} + + +# Computes the results file for a test suite run. +# +# The computed path is "generic" in the sense that it does not include a +# real timestamp: it only includes a placeholder. This function should be +# used along the utils_strip_times function so that the timestampts of the +# real results files are stripped out. +# +# \param path Optional path to use; if not given, use the cwd. +utils_results_file() { + echo "${HOME}/.kyua/store/results.$(utils_results_id "${@}").db" +} + + +# Copies a helper binary from the source directory to the work directory. +# +# \param name The name of the binary to copy. +# \param destination The target location for the binary; can be either +# a directory name or a file name. +utils_cp_helper() { + local name="${1}"; shift + local destination="${1}"; shift + + ln -s "$(atf_get_srcdir)"/helpers/"${name}" "${destination}" +} + + +# Creates a 'kyua' binary in the path that strips timing data off the output. +# +# Call this on test cases that wish to replace timing data in the *stdout* of +# Kyua with the deterministic strings. This is to be used by tests that +# validate the 'test' and 'report' subcommands. +utils_install_times_wrapper() { + [ ! -x kyua ] || return + cat >kyua <<EOF +#! /bin/sh + +PATH=${PATH} + +kyua "\${@}" >kyua.tmpout +result=\${?} +cat kyua.tmpout | ${utils_strip_times} +exit \${result} +EOF + chmod +x kyua + PATH="$(pwd):${PATH}" +} + + +# Creates a 'kyua' binary in the path that makes the output of 'test' stable. +# +# Call this on test cases that wish to replace timing data with deterministic +# strings and that need the result lines in the output to be sorted +# lexicographically. The latter hides the indeterminism caused by parallel +# execution so that the output can be verified. For these reasons, this is to +# be used exclusively by tests that validate the 'test' subcommand. +utils_install_stable_test_wrapper() { + [ ! -x kyua ] || return + cat >kyua <<EOF +#! /bin/sh + +PATH=${PATH} + +kyua "\${@}" >kyua.tmpout +result=\${?} +cat kyua.tmpout | ${utils_strip_times} >kyua.tmpout2 + +# Sort the test result lines but keep the rest intact. +grep '[^ ]*:[^ ]*' kyua.tmpout2 | sort >kyua.tmpout3 +grep -v '[^ ]*:[^ ]*' kyua.tmpout2 >kyua.tmpout4 +cat kyua.tmpout3 kyua.tmpout4 + +exit \${result} +EOF + chmod +x kyua + PATH="$(pwd):${PATH}" +} + + +# Defines a test case with a default head. +utils_test_case() { + local name="${1}"; shift + + atf_test_case "${name}" + eval "${name}_head() { + atf_set require.progs kyua + }" +} + + +# Computes the test suite identifier for results files files. +# +# \param path Optional path to use; if not given, use the cwd. +utils_test_suite_id() { + local path= + if [ ${#} -gt 0 ]; then + path="$(cd ${1} && pwd)"; shift + else + path="$(pwd)" + fi + echo "${path}" | sed -e 's,^/,,' -e 's,/,_,g' +} |