Index: tools/tools/smoketestsuite/Makefile =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/Makefile @@ -0,0 +1,31 @@ +# $FreeBSD$ +# +# Makefile for building the test generation tool + +PROG_CXX= generate_tests +LOCALBASE= /usr/local +MAN= +CXXFLAGS+= -I${LOCALBASE}/include -std=c++11 +LDFLAGS+= -L${LOCALBASE}/lib -lboost_filesystem -lboost_system +SRCS= logging.cpp \ + utils.cpp \ + read_annotations.cpp \ + generate_license.cpp \ + add_testcase.cpp \ + fetch_groff.cpp \ + generate_test.cpp + +.PHONY: clean \ + fetch_utils \ + run + +fetch_utils: + sh ${.CURDIR}/scripts/fetch_utils.sh + +run: + @echo Generating annotations... + sh ${.CURDIR}/scripts/generate_annot.sh + @echo Generating test files... + ./generate_tests + +.include Index: tools/tools/smoketestsuite/README =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/README @@ -0,0 +1,50 @@ +$FreeBSD$ + +Smoke testing of base utilities +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Test generation tool made as a part of Google Summer of Code '17 with FreeBSD (https://summerofcode.withgoogle.com/projects/#6426676740227072). +Refer the FreeBSD wiki (https://wiki.freebsd.org/SummerOfCode2017/SmokeTestingOfBaseUtilities) for an overview and updates. +A brief description and implementation details: https://lists.freebsd.org/pipermail/soc-status/2017-July/001079.html. +The diagram "architecture.png" briefly summarizes how different components fit with the testcase-generator. + +Project layout +~~~~~~~~~~~~~~ + +. +├── annotations +│   └── ........................:: Annotation files (generated/user-defined) +├── scripts +│   └── ........................:: Helper scripts +├── architecture.png ...........:: A brief architecture diagram +├── add_testcase.cpp ...........:: Testcase generator +├── generate_license.cpp .......:: Customized license generator +├── generate_test.cpp ..........:: Test generator +├── logging.cpp ................:: Logger +├── read_annotations.cpp .......:: Annotation parser +└── utils.cpp ..................:: Index generator + +- - - + +Dependencies +~~~~~~~~~~~~ +* Boost C++ libraries: The tool was tested to work with the port "boost-all-1.64.0". +* Kyua + +Instructions +~~~~~~~~~~~~ +* The tool needs to know about the utilities in src which don't already have tests. The list of all such utilities can be generated via - + + make fetch_utils + + This will generate the file "scripts/utils_list". + +* For generating the tests, execute the following - + + make clean + make && make run + +ToDo +~~~~ +The following features/functionalities are planned to be integrated - +* [Batch mode] Appropriately update `BSD.tests.dist`. Index: tools/tools/smoketestsuite/add_testcase.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/add_testcase.h @@ -0,0 +1,43 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ADD_TESTCASE_H_ +#define _ADD_TESTCASE_H_ + +namespace addtestcase { + void KnownTestcase(std::string, std::string, std::string, \ + std::string, std::ofstream&); + + void UnknownTestcase(std::string, std::string, std::pair, \ + std::string&, bool); + + void NoArgsTestcase(std::string, std::pair, \ + std::ofstream&, bool); +} + +#endif /* _ADD_TESTCASE_H_ */ Index: tools/tools/smoketestsuite/add_testcase.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/add_testcase.cpp @@ -0,0 +1,177 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include + +#include "add_testcase.h" + +/* Adds a test-case for an option with known usage. */ +void +addtestcase::KnownTestcase(std::string option, + std::string util_with_section, + std::string descr, + std::string output, + std::ofstream& test_script) +{ + std::string testcase_name; + std::string utility = util_with_section.substr(0, + util_with_section.size() - 3); + + /* Add testcase name. */ + test_script << "atf_test_case "; + if (!option.empty()) { + testcase_name = option; + testcase_name.append("_flag"); + } else + testcase_name = "no_arguments"; + test_script << testcase_name + "\n"; + + /* Add testcase description. */ + test_script << testcase_name + + "_head()\n{\n\tatf_set \"descr\" "; + if (!descr.empty()) + test_script << descr; + else + test_script << "\"Verify the usage of option \'" + + option + "\'\""; + test_script << "\n}\n\n"; + + /* Add body of the testcase. */ + test_script << testcase_name + "_body()\n{" + + "\n\tatf_check -s exit:0 -o "; + + /* Match the usage output if generated. */ + if (!output.empty()) + test_script << "inline:\"" + output + "\" "; + else + test_script << "empty "; + test_script << utility; + + if (!option.empty()) + test_script << " -" + option; + test_script << "\n}\n\n"; +} + +/* Adds a test-case for an option with unknown usage. */ +void +addtestcase::UnknownTestcase(std::string option, + std::string util_with_section, + std::pair output, + std::string& testcase_buffer, + bool usage_output) +{ + std::string utility = util_with_section.substr(0, + util_with_section.size() - 3); + + if (output.second) + testcase_buffer.append("\n\tatf_check -s not-exit:0 -e "); + else + testcase_buffer.append("\n\tatf_check -s exit:0 -o "); + + /* Check if a usage message was produced (case-insensitive match). */ + if (usage_output) + testcase_buffer.append("match:\"$usage_output\" "); + else if (!output.first.empty()) + testcase_buffer.append("inline:\"" + output.first + "\" "); + else + testcase_buffer.append("empty "); + + testcase_buffer.append(utility); + + if (!option.empty()) + testcase_buffer.append(" -" + option); +} + +/* Adds a test-case for usage without any arguments. */ +void +addtestcase::NoArgsTestcase(std::string util_with_section, + std::pair output, + std::ofstream& test_script, + bool usage_output) +{ + std::string descr; + std::string utility = util_with_section.substr(0, + util_with_section.size() - 3); + + if (output.second) { + /* An error was encountered. */ + test_script << std::string("atf_test_case no_arguments\n") + + "no_arguments_head()\n{\n\tatf_set \"descr\" "; + if (!output.first.empty()) { + /* + * We expect a usage message to be generated in this + * case (case-insensitive match). + */ + if (usage_output) { + descr = "\"Verify that " + util_with_section + + " fails and generates a valid usage \" " + + "\\\n\t\t\t\"message when no arguments " + + "are supplied\""; + + test_script << descr + + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e match:" + + "\"$usage_output\" " + utility; + } else { + descr = "\"Verify that " + util_with_section + + " fails and generates a valid output \" " + + "\\\n\t\t\t\"when no arguments are supplied\""; + + test_script << descr + + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e inline:" + + "\"" + output.first + "\" " + + utility; + } + } else { + descr = "\"Verify that " + util_with_section + " fails " + + "silently when no arguments are supplied\"" ; + test_script << descr + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e empty " + + utility; + } + test_script << "\n}\n\n"; + } else { + /* + * The command ran successfully, hence we guessed + * a correct usage for the utility under test. + */ + if (!output.first.empty()) + descr = "\"Verify that " + util_with_section + " executes " + + "successfully and produces a valid \" \\\n\t\t\t" + + "\"output when invoked without any arguments\""; + else + descr = "\"Verify that " + util_with_section + + " executes successfully and silently \" \\\n" + + "\t\t\t\"when invoked without any arguments\""; + + addtestcase::KnownTestcase("", util_with_section, descr, + output.first, test_script); + } +} Index: tools/tools/smoketestsuite/annotations/date_test.ant =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/annotations/date_test.ant @@ -0,0 +1,5 @@ +R_flag +j_flag +n_flag +no_arguments +u_flag Index: tools/tools/smoketestsuite/fetch_groff.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/fetch_groff.h @@ -0,0 +1,39 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _FETCH_GROFF_H_ +#define _FETCH_GROFF_H_ + +#include + +namespace groff { + extern std::unordered_map groff_map; + int FetchGroffScripts(); +} + +#endif /* _FETCH_GROFF_H_ */ Index: tools/tools/smoketestsuite/fetch_groff.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/fetch_groff.cpp @@ -0,0 +1,98 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "fetch_groff.h" +#include "logging.h" + +/* Map of utility name and its location in src tree. */ +std::unordered_map groff::groff_map; + +/* + * Traverses the FreeBSD src tree looking for groff scripts for section + * 1 and section 8 utilities and stores their location in a hashmap. + */ +int +groff::FetchGroffScripts() +{ + std::string utils_list = "scripts/utils_list"; + std::string src = "../../../"; /* FreeBSD src. */ + std::string utildir; + std::string utilname; + std::string path; + std::ifstream file; + std::regex section ("(.*).(?:1|8)"); + struct stat sb; + struct dirent *ent; + DIR *dir; + + /* Check if the file "scripts/utils_list" exists. */ + if (stat(utils_list.c_str(), &sb) != 0) { + std::cerr << "scripts/utils_list does not exists.\n" + "Run 'make fetch_utils' first.\n"; + return EXIT_FAILURE; + } + file.open(utils_list); + + while (getline(file, utildir)) { + path = src + utildir + "/tests"; + /* + * Copy the groff script only if the utility does not + * already have tests, i.e. the "tests" directory is absent. + */ + if (stat(path.c_str(), &sb) || !S_ISDIR(sb.st_mode)) { + path = src + utildir + "/"; + utilname = utildir.substr(utildir.find_last_of("/") + 1); + if ((dir = opendir(path.c_str())) != NULL) { + /* Skip directory entry for "." and "..". */ + ent = readdir(dir); + ent = readdir(dir); + while ((ent = readdir(dir)) != NULL) { + if (std::regex_match(ent->d_name, section)) { + groff_map[utilname] = path + ent->d_name; + } + } + closedir(dir); + } else { + logging::LogPerror("opendir()"); + } + } + } + + file.close(); + return EXIT_SUCCESS; +} Index: tools/tools/smoketestsuite/generate_license.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_license.h @@ -0,0 +1,36 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _GENERATE_LICENSE_H_ +#define _GENERATE_LICENSE_H_ + +namespace generatelicense { + std::string GenerateLicense(int, char **); +} + +#endif /* _GENERATE_LICENSE_H_ */ Index: tools/tools/smoketestsuite/generate_license.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_license.cpp @@ -0,0 +1,82 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include "generate_license.h" +#include "utils.h" + +std::string +generatelicense::GenerateLicense(int argc, char **argv) +{ + std::string license; + std::string copyright_owner; + + if (argc > 1) { + if (argc == 3 && strncmp(argv[1], "--name ", 7)) + copyright_owner = argv[2]; + else { + std::cerr << "Usage: ./generate_tests --name \n"; + exit(EXIT_FAILURE); + } + } else + copyright_owner = utils::Execute("id -P | cut -d : -f 8").first; + + license = + "#\n" + "# Copyright 2017-2018 " + copyright_owner + "\n" + "# All rights reserved.\n" + "#\n" + "# Redistribution and use in source and binary forms, with or without\n" + "# modification, are permitted provided that the following conditions\n" + "# are met:\n" + "# 1. Redistributions of source code must retain the above copyright\n" + "# notice, this list of conditions and the following disclaimer.\n" + "# 2. Redistributions in binary form must reproduce the above copyright\n" + "# notice, this list of conditions and the following disclaimer in the\n" + "# documentation and/or other materials provided with the distribution.\n" + "#\n" + "# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND\n" + "# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n" + "# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n" + "# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n" + "# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n" + "# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n" + "# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n" + "# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n" + "# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n" + "# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n" + "# SUCH DAMAGE.\n" + "#\n" + "# $FreeBSD$\n" + "#\n\n"; + + return license; +} Index: tools/tools/smoketestsuite/generate_test.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_test.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _GENERATE_TEST_H_ +#define _GENERATE_TEST_H_ + +#include "utils.h" + +namespace generatetest { + void IntHandler(int); + void GenerateMakefile(std::string, std::string); + void GenerateTest(std::string, char, + std::string&, const char*); +} + +#endif /* _GENERATE_TEST_H_ */ Index: tools/tools/smoketestsuite/generate_test.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_test.cpp @@ -0,0 +1,404 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "add_testcase.h" +#include "fetch_groff.h" +#include "generate_license.h" +#include "generate_test.h" +#include "logging.h" +#include "read_annotations.h" + +void +generatetest::IntHandler(int dummmy) +{ + std::cerr << "\nExiting...\n"; + /* Remove the temporary directory. */ + boost::filesystem::remove_all(utils::tmpdir); + exit(EXIT_FAILURE); +} + +/* [Batch mode] Generate a makefile for the test of given utility. */ +void +generatetest::GenerateMakefile(std::string utility, std::string utildir) +{ + std::ofstream file; + + file.open(utildir + "/Makefile", std::ios::out); + file << "# $FreeBSD$\n\nATF_TESTS_SH+= " + + utility + "_test\n\n" + + ".include \n"; + file.close(); +} + +/* Generate a test for the given utility. */ +void +generatetest::GenerateTest(std::string utility, + char section, + std::string& license, + const char *testsdir) +{ + std::vector usage_messages; + std::vector identified_opts; + std::string command; + std::string testcase_list; + std::string buffer; + std::string testfile; + std::string util_with_section; + std::ofstream file; + std::pair output; + std::unordered_set annotation_set; + /* Number of options for which a testcase has been generated. */ + int progress = 0; + bool usage_output = false; /* Tracks whether '$usage_output' variable is used. */ + + /* Read annotations and populate hash set "annotation_set". */ + annotations::read_annotations(utility, annotation_set); + util_with_section = utility + '(' + section + ')'; + utils::OptDefinition opt_def; + identified_opts = opt_def.CheckOpts(utility); + testfile = testsdir + utility + "_test.sh"; + +#ifndef DEBUG + /* Indicate the start of test generation for current utility. */ + if (isatty(fileno(stderr))) { + std::cerr << std::setw(18) << util_with_section << " | " + << progress << "/" << opt_def.opt_list.size() << "\r"; + } +#endif + /* Add license in the generated test scripts. */ + file.open(testfile, std::ios::out); + file << license; + + /* + * If a known option was encountered (i.e. `identified_opts` is + * populated), produce a testcase to check the validity of the + * result of that option. If no known option was encountered, + * produce testcases to verify the correct (generated) usage + * message when using the supported options incorrectly. + */ + for (const auto &i : identified_opts) { + command = utils::GenerateCommand(utility, i->value); + output = utils::Execute(command); + if (boost::iequals(output.first.substr(0, 6), "usage:")) { + /* Our guessed usage is incorrect as usage message is produced. */ + addtestcase::UnknownTestcase(i->value, util_with_section, + output, buffer, usage_output); + } else { + addtestcase::KnownTestcase(i->value, util_with_section, + "", output.first, file); + } + testcase_list.append("\tatf_add_test_case " + i->value + "_flag\n"); + } + + /* Add testcases for the options whose usage is not yet known. + * For the purpose of adding a "$usage_output" variable, + * we choose the option which produces one. + * TODO Avoid double executions of an option, i.e. one while + * selecting usage message and another while generating testcase. + */ + if (opt_def.opt_list.size() == 1) { + /* Check if the single option produces a usage message. */ + command = utils::GenerateCommand(utility, opt_def.opt_list.front()); + output = utils::Execute(command); + if (output.second && !output.first.empty()) { + usage_output = true; + file << "usage_output=\'" + output.first + "\'\n\n"; + } + } else if (opt_def.opt_list.size() > 1) { + /* + * Utility supports multiple options. In case the usage message + * is consistent for atleast "two" options, we reduce duplication + * by assigning a variable "usage_output" in the test script. + */ + for (const auto &i : opt_def.opt_list) { + command = utils::GenerateCommand(utility, i); + output = utils::Execute(command); + if (output.second && usage_messages.size() < 3) + usage_messages.push_back(output.first); + } + + for (int j = 0; j < usage_messages.size(); j++) { + if (!usage_messages[j].compare + (usage_messages[(j+1) % usage_messages.size()])) { + usage_output = true; + file << "usage_output=\'" + + output.first.substr(0, 7 + utility.size()) + + "\'\n\n"; + break; + } + } + } + + /* + * Execute the utility with supported options, while + * adding positive and negative testcases accordingly. + */ + for (const auto &i : opt_def.opt_list) { + /* Ignore the option if it is annotated. */ + if (annotation_set.find(i) != annotation_set.end()) + continue; + + command = utils::GenerateCommand(utility, i); + output = utils::Execute(command); +#ifndef DEBUG + if (isatty(fileno(stderr))) { + std::cerr << std::setw(18) << util_with_section + << " | " << ++progress << "/" + << opt_def.opt_list.size() << "\r"; + } +#endif + if (output.second) { + addtestcase::UnknownTestcase(i, util_with_section, output, + buffer, usage_output); + } else { + /* Guessed usage is correct as EXIT_SUCCESS is encountered */ + addtestcase::KnownTestcase(i, util_with_section, "", + output.first, file); + testcase_list.append(std::string("\tatf_add_test_case ") + + i + "_flag\n"); + } + } + std::cout << std::endl; /* Takes care of the last '\r'. */ + + if (!opt_def.opt_list.empty()) { + testcase_list.append("\tatf_add_test_case invalid_usage\n"); + file << "atf_test_case invalid_usage\ninvalid_usage_head()\n" + << "{\n\tatf_set \"descr\" \"Verify that an invalid usage " + << "with a supported option \" \\\n\t\t\t\"produces a valid " + << "error message\"\n}\n\ninvalid_usage_body()\n{"; + + file << buffer + "\n}\n\n"; + } + + /* + * Add a testcase under "no_arguments" for + * running the utility without any arguments. + */ + if (annotation_set.find("*") == annotation_set.end()) { + command = utils::GenerateCommand(utility, ""); + output = utils::Execute(command); + addtestcase::NoArgsTestcase(util_with_section, output, + file, usage_output); + testcase_list.append("\tatf_add_test_case no_arguments\n"); + } + + file << "atf_init_test_cases()\n{\n" + testcase_list + "}\n"; + file.close(); +} + +int +main(int argc, char **argv) +{ + std::ifstream groff_list; + struct stat sb; + struct dirent *ent; + char answer; + std::string license; + std::string utildir; /* Path to utility in src tree. */ + std::string groffpath; + const char *testsdir = "generated_tests/"; + /* + * Instead of generating tests for all the utilities, "batch mode" + * allows generation of tests for first "batch_limit" number of + * utilities selected from "scripts/utils_list". + */ + bool batch_mode = false; + int batch_limit; /* Number of tests to be generated in batch mode. */ + + /* Handle interrupts. */ + signal(SIGINT, generatetest::IntHandler); + + if (groff::FetchGroffScripts() == EXIT_FAILURE) + return EXIT_FAILURE; + + /* + * Create a temporary directory where all the side-effects + * introduced by utility-specific commands are restricted. + */ + boost::filesystem::create_directory(utils::tmpdir); + + std::cout << "\nInstead of generating tests for all the utilities, 'batch mode'\n" + "allows generation of tests for first few utilities selected from\n" + "'scripts/utils_list', and places them at their correct location\n" + "in the src tree, with corresponding makefiles created.\n" + "NOTE: You will be prompted for the superuser password when\n" + "creating test directory under '/usr/tests/' and when installing\n" + "the tests via `sudo make install`.\n" + "Run in 'batch mode' ? [y/N] "; + std::cin.get(answer); + + switch(answer) { + case 'y': + case 'Y': + batch_mode = true; + std::cout << "Number of utilities to select for test generation: "; + std::cin >> batch_limit; + + if (batch_limit <= 0) { + std::cerr << "Invalid input. Exiting...\n"; + return EXIT_FAILURE; + } + break; + case '\n': + default: + break; + } + + /* Check if the directory "testsdir" exists. */ + if (stat(testsdir, &sb) || !S_ISDIR(sb.st_mode)) { + boost::filesystem::path dir(testsdir); + if (boost::filesystem::create_directory(dir)) + std::cout << "Directory created: " << testsdir << "\n"; + else { + std::cerr << "Unable to create directory: " << testsdir << "\n"; + return EXIT_FAILURE; + } + } + + /* Generate a license to be added in the generated scripts. */ + license = generatelicense::GenerateLicense(argc, argv); + +#ifndef DEBUG + /* Generate a tabular-like format. */ + std::cout << std::endl; + std::cout << std::setw(30) << "Utility | Progress\n"; + std::cout << std::setw(32) << "----------+-----------\n"; +#endif + + if (batch_mode) { + /* Number of hops required to reach root directory. */ + std::string hops = "../../../../"; + std::string command; + std::string installdir; /* Directory where tests are installed. */ + int retval; + int maxdepth = 32; + std::unordered_map::iterator it; + /* Remember pwd so that we can return back. */ + boost::filesystem::path tooldir = boost::filesystem::current_path(); + + /* + * Discover number of hops required to reach root directory. + * To avoid an infinite loop in case directory "usr/" doesn't + * exist in the filesystem, we assume that the maximum depth + * from root directory at which pwd is located is "maxdepth". + */ + chdir(hops.c_str()); /* Move outside FreeBSD src. */ + while (maxdepth--) { + if (chdir("..") == -1) { + perror("chdir"); + return EXIT_FAILURE; + } + hops += "../"; + if (stat("usr", &sb) == 0 && S_ISDIR(sb.st_mode)) + break; + } + + /* + * Generate tests for first "batch_limit" number of + * utilities selected from "scripts/utils_list". + */ + it = groff::groff_map.begin(); + while (batch_limit-- && it != groff::groff_map.end()) { + /* Move back to the tool's directory. */ + boost::filesystem::current_path(tooldir); + + groffpath = groff::groff_map.at(it->first); + utildir = groffpath.substr + (0, groffpath.size() - 2 - it->first.size()); + installdir = hops + "usr/tests/" + utildir.substr(9); + utildir += "tests/"; + + /* Populate "tests/" directory. */ + boost::filesystem::remove_all(utildir); + boost::filesystem::create_directory(utildir); + generatetest::GenerateMakefile(it->first, utildir); + generatetest::GenerateTest(it->first, it->second.back(), + license, utildir.c_str()); + boost::filesystem::copy_file(utildir + it->first + "_test.sh", + testsdir + it->first + "_test.sh", + boost::filesystem::copy_option::overwrite_if_exists); + std::advance(it, 1); + + /* Execute the generated test and note success/failure. */ + if (stat(installdir.c_str(), &sb) || !S_ISDIR(sb.st_mode)) { + command = "sudo mkdir -p " + installdir; + if ((retval = system(command.c_str())) == -1) { + perror("system"); + return EXIT_FAILURE; + } else if (retval) { + boost::filesystem::current_path(tooldir); + boost::filesystem::remove_all(utildir); + continue; + } + } + + /* Install the test. */ + chdir(utildir.c_str()); + if ((retval = system("sudo make install")) == -1) { + perror("system"); + return EXIT_FAILURE; + } else if (retval) { + boost::filesystem::current_path(tooldir); + boost::filesystem::remove_all(utildir); + continue; + } + boost::filesystem::current_path(tooldir); + + /* Run the test. */ + chdir(installdir.c_str()); + if ((retval = system("kyua test")) == -1) { + perror("system"); + return EXIT_FAILURE; + } else if (retval) { + boost::filesystem::current_path(tooldir); + boost::filesystem::remove_all(utildir); + continue; + } + } + } else { + for (const auto &it : groff::groff_map) { + generatetest::GenerateTest(it.first, it.second.back(), + license, testsdir); + } + } + + /* Cleanup. */ + boost::filesystem::remove_all(utils::tmpdir); + return EXIT_SUCCESS; +} Index: tools/tools/smoketestsuite/logging.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/logging.h @@ -0,0 +1,45 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LOGGING_H_ +#define _LOGGING_H_ + +/* Use a gcc variadic macro to conditionally compile debug printing. */ +#ifdef DEBUG +#define DEBUGP(...) \ + fprintf(stdout, __VA_ARGS__); \ + fflush(stdout); +#else +#define DEBUGP(...) {} +#endif /* DEBUG */ + +namespace logging { + void LogPerror(const char *); +} + +#endif /* _LOGGING_H_ */ Index: tools/tools/smoketestsuite/logging.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/logging.cpp @@ -0,0 +1,39 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include + +#include "logging.h" + +void +logging::LogPerror(const char *message) +{ +#ifdef DEBUG + perror(message); +#endif +} Index: tools/tools/smoketestsuite/read_annotations.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/read_annotations.h @@ -0,0 +1,39 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _READ_ANNOTATIONS_H_ +#define _READ_ANNOTATIONS_H_ + +#include + +namespace annotations { + void read_annotations(std::string, \ + std::unordered_set&); +} + +#endif /* _READ_ANNOTATIONS_H_ */ Index: tools/tools/smoketestsuite/read_annotations.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/read_annotations.cpp @@ -0,0 +1,58 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include "read_annotations.h" + +/* Read the annotation files and skip generation of respective tests. */ +void +annotations::read_annotations(std::string utility, + std::unordered_set& annotation_set) +{ + std::string line; + std::ifstream file; + file.open("annotations/" + utility + "_test.annot"); + + while (getline(file, line)) { + /* Add a unique identifier for no_arguments testcase */ + if (!line.compare(0, 12, "no_arguments")) + annotation_set.insert("*"); + /* + * Add flag value for supported argument testcases + * Doing so we ignore the "invalid_usage" testcase + * as it is guaranteed to always succeed. + */ + else if (!line.compare(2, 4, "flag")) + annotation_set.insert(line.substr(0, 1)); + } + + file.close(); +} Index: tools/tools/smoketestsuite/scripts/README =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/README @@ -0,0 +1,9 @@ +Helper scripts +~~~~~~~~~~~~~~ + +Script Name | Functionality +------------------+----------------- +fetch_groff.sh | Lists location of all section 1 utilities +fetch_utils.sh | Saves all the base utilities in the src tree in **utils_list** +generate_annot.sh | Populates annotation files under [annotations](../annotations) +validate.sh | Validates side-effects of newly introduced changes in the tool Index: tools/tools/smoketestsuite/scripts/fetch_utils.sh =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/fetch_utils.sh @@ -0,0 +1,44 @@ +#! /bin/sh +# +# Copyright 2017-2018 Shivansh Rai +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +# Script for listing all the base utilities + +set -eu + +script_dir="$(dirname $0)" +src="$(dirname $0)/../../../../" + +fetch_utils() { + cd "$src" + find . -name Makefile | xargs grep -l 'PROG\|PROG_CXX' \ + | sed -e 's|/Makefile$||' | cut -c 3- +} + +rm -f "$script_dir/utils_list" + +(fetch_utils) >> "$script_dir/utils_list" Index: tools/tools/smoketestsuite/scripts/generate_annot.sh =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/generate_annot.sh @@ -0,0 +1,107 @@ +#!/bin/sh +# +# Copyright 2017-2018 Shivansh Rai +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +# Script for generating annotations based on generated tests. + +set -eu + +update_annotations() { + cd "$testdir" 2>/dev/null + report=$(kyua report) + cd - + i=2 + annotations="" + diff="" + + while true + do + testcase=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $1}') + exitstatus=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $3}') + check=$(printf "%s" "$testcase" | cut -s -f1 -d":") + + if [ ! "$check" ] && [ "$annotations" ]; then + file="$tooldir/annotations/$test.ant" # Annotations file + # Append only the new annotations. + printf "$annotations" > "$file.temp" + [ ! -e "$file" ] && touch "$file" + diff=$(comm -13 "$file" "$file.temp") + if [ "$diff" ]; then + if [ $prompt = 0 ]; then + prompt=1 + printf "\nModified annotation files -\n" + fi + printf "$diff\n" >> "$file" + printf " * %s\n" "$test.ant" + fi + rm -f "$file.temp" + break + fi + + if [ "$exitstatus" = "failed:" ]; then + testcase=${testcase#"$test:"} + annotations="$annotations$testcase\n" + elif [ ! "$exitstatus" ]; then + break + fi + i=$((i+1)) + done +} + +suffix="_test" +extension=".sh" +prompt=0 +tooldir=$(dirname $0)/.. + +# Check if the directory "generated_tests/" is populated. +if [ ! -d "$tooldir/generated_tests" ] || \ + [ -z "$(ls -A $tooldir/generated_tests)" ]; then + exit +fi + +for f in "$tooldir/generated_tests"/* +do + file=$(basename "$f") + test=${file%$extension} + utility=${test%$suffix} + + # Guess the correct installation directory. + if [ -d "/usr/tests/bin/$utility" ]; then + testdir="/usr/tests/bin/$utility" + elif [ -d "/usr/tests/usr.bin/$utility" ]; then + testdir="/usr/tests/usr.bin/$utility" + elif [ -d "/usr/tests/usr.sbin/$utility" ]; then + testdir="/usr/tests/usr.sbin/$utility" + else + continue + fi + update_annotations +done + +if [ $prompt = 1 ]; then + printf "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n" +fi Index: tools/tools/smoketestsuite/utils.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/utils.h @@ -0,0 +1,83 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _UTILS_H_ +#define _UTILS_H_ + +#include +#include + +namespace utils { + /* + * Option relation which maps option names to + * a unique identifier in their description. + */ + struct OptRelation { + char type; /* Option type: (s)short/(l)long. */ + std::string value; /* Name of the option. */ + /* The keyword which should be looked up in usage + * message (if) produced when using this options. + */ + std::string keyword; + }; + + /* + * Read/Write file descriptors for a pipe. + */ + struct PipeDescriptor { + int readfd; + int writefd; + pid_t pid; /* PID of the forked shell process. */ + }; + + /* + * Temporary directory inside which the utility-specific + * commands will be executed, and all the side effects + * (core dumps, executables etc.) that are created will + * be restricted in this directory. + */ + extern const char *tmpdir; + + std::string GenerateCommand(std::string, std::string); + std::pair Execute(std::string); + PipeDescriptor* POpen(const char*); + + class OptDefinition { + public: + /* List of all the accepted options with unknown usage. */ + std::vector opt_list; + /* Map "option value" to "option definition". */ + std::unordered_map opt_map; + std::unordered_map::iterator opt_map_iter; + + void InsertOpts(); + std::vector CheckOpts(std::string); + }; +} + +#endif /* _UTILS_H_ */ Index: tools/tools/smoketestsuite/utils.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/utils.cpp @@ -0,0 +1,336 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "utils.h" +#include "fetch_groff.h" +#include "logging.h" + +#define READ 0 /* Pipe descriptor: read end. */ +#define WRITE 1 /* Pipe descriptor: write end. */ +/* + * Buffer size (used for buffering output generated + * after executing the utility-specific command). + */ +#define BUFSIZE 128 +#define TIMEOUT 1 /* Threshold (seconds) for a function call to return. */ + +const char *utils::tmpdir = "tmpdir"; +/* + * Insert a list of user-defined option definitions + * into a hashmap. These specific option definitions + * are the ones which one can be easily tested. + */ +void +utils::OptDefinition::InsertOpts() +{ + /* Option definitions. */ + OptRelation h_def; /* '-h' */ + h_def.type = 's'; + h_def.value = "h"; + h_def.keyword = "help"; + + OptRelation v_def; /* '-v' */ + v_def.type = 's'; + v_def.value = "v"; + v_def.keyword = "version"; + + /* + * "opt_map" contains all the options + * which can be easily tested. + */ + opt_map.insert(std::make_pair + ("h", (OptRelation)h_def)); + opt_map.insert(std::make_pair + ("v", (OptRelation)v_def)); +}; + +/* + * For the utility under test, find the supported options + * present in the hashmap generated by InsertOpts() + * and return them in a form of list of option relations. + */ +std::vector +utils::OptDefinition::CheckOpts(std::string utility) +{ + std::string opt_id = ".It Fl"; /* Option identifier in man page. */ + std::string line; /* An individual line in a man-page. */ + std::string opt_name; /* Name of the option. */ + std::string buffer; /* Option description extracted from man-page. */ + std::string opt_string; /* Identified option names. */ + int opt_pos; /* Starting index of the (identified) option. */ + int space_index; /* First occurrence of space in option definition. */ + std::vector identified_opts; + std::vector supported_sections = { "1", "8" }; + + /* Generate the hashmap "opt_map". */ + InsertOpts(); + std::ifstream infile(groff::groff_map[utility]); + + /* + * Search for all the options accepted by the + * utility and collect those present in "opt_map". + */ + while (std::getline(infile, line)) { + if ((opt_pos = line.find(opt_id)) != std::string::npos) { + /* Locate the position of option name. */ + opt_pos += opt_id.size() + 1; + if (opt_pos > line.size()) { + /* + * This condition will trigger when a utility + * supports an empty argument, e.g. tset(issue #9) + */ + continue; + } + + /* + * Check for long options ; While here, also sanitize + * multi-word option definitions in a man page to properly + * extract short options from option definitions such as: + * .It Fl r Ar seconds (taken from date(1)). + */ + if ((space_index = line.find(" ", opt_pos + 1, 1)) + != std::string::npos) + opt_name = line.substr(opt_pos, space_index - opt_pos); + else + opt_name = line.substr(opt_pos); + + /* + * Check if the identified option matches the identifier. + * "opt_list.back()" is the previously checked option, the + * description of which is now stored in "buffer". + */ + if (!opt_list.empty() && + (opt_map_iter = opt_map.find(opt_list.back())) + != opt_map.end() && + buffer.find((opt_map_iter->second).keyword) != std::string::npos) { + identified_opts.push_back(&(opt_map_iter->second)); + /* + * Since the usage of the option under test + * is known, we remove it from "opt_list". + */ + opt_list.pop_back(); + } + + /* Update the list of valid options. */ + opt_list.push_back(opt_name); + /* Empty the buffer for next option's description. */ + buffer.clear(); + } else { + /* + * Collect the option description until next + * valid option definition is encountered. + */ + buffer.append(line); + } + } + + return identified_opts; +} + +/* Generates command for execution. */ +std::string +utils::GenerateCommand(std::string utility, std::string opt) +{ + std::string command = utility; + + if (!opt.empty()) + command += " -" + opt; + command += " 2>&1 readfd = pdes[READ]; + pipe_descr->writefd = pdes[WRITE]; + + /* Type-cast to avoid compiler warnings [-Wwrite-strings]. */ + argv[0] = (char *)"sh"; + argv[1] = (char *)"-c"; + argv[2] = (char *)command; + argv[3] = NULL; + + switch (child_pid = vfork()) { + case -1: /* Error. */ + free(pipe_descr); + close(pdes[READ]); + close(pdes[WRITE]); + return NULL; + case 0: /* Child. */ + /* + * TODO Verify if the following operations on both read + * and write file-descriptors (irrespective of the value + * of 'type', which is no longer being used) is safe. + */ + if (pdes[WRITE] != STDOUT_FILENO) + dup2(pdes[WRITE], STDOUT_FILENO); + else + fcntl(pdes[WRITE], F_SETFD, 0); + if (pdes[READ] != STDIN_FILENO) + dup2(pdes[READ], STDIN_FILENO); + else + fcntl(pdes[READ], F_SETFD, 0); + + /* + * For current usecase, it might so happen that the child gets + * stuck on a blocking read (e.g. passwd(1)) waiting for user + * input. In that case the child will be killed via a signal. + * To avoid any effect on the parent's execution, we place the + * child in a separate process group with pgid set as "child_pid". + */ + setpgid(child_pid, child_pid); + execve("/bin/sh", argv, NULL); + exit(127); + } + + pipe_descr->pid = child_pid; + return pipe_descr; +} + +/* + * Executes the command passed as argument in a + * shell and returns its output and exit status. + */ +std::pair +utils::Execute(std::string command) +{ + int result; + int exitstatus; + std::array buffer; + std::string usage_output; + struct timeval tv; + fd_set readfds; + FILE *pipe; + PipeDescriptor *pipe_descr; + pid_t pid; + int pstat; + + /* Execute "command" inside "tmpdir". */ + chdir(tmpdir); + pipe_descr = utils::POpen(command.c_str()); + chdir(".."); + + if (pipe_descr == NULL) { + logging::LogPerror("utils::POpen()"); + exit(EXIT_FAILURE); + } + + /* Close the unrequired file-descriptor. */ + close(pipe_descr->writefd); + + pipe = fdopen(pipe_descr->readfd, "r"); + free(pipe_descr); + if (pipe == NULL) { + close(pipe_descr->readfd); + logging::LogPerror("fdopen()"); + exit(EXIT_FAILURE); + } + + /* Set a timeout for shell process to complete its execution. */ + tv.tv_sec = TIMEOUT; + tv.tv_usec = 0; + + FD_ZERO(&readfds); + FD_SET(fileno(pipe), &readfds); + result = select(fileno(pipe) + 1, &readfds, NULL, NULL, &tv); + + if (result > 0) { + while (!feof(pipe) && !ferror(pipe)) { + if (fgets(buffer.data(), BUFSIZE, pipe) != NULL) + usage_output += buffer.data(); + } + } else if (result == -1) { + logging::LogPerror("select()"); + if (kill(pipe_descr->pid, SIGTERM) < 0) + logging::LogPerror("kill()"); + } else if (result == 0) { + /* + * We gave a relaxed value of TIMEOUT seconds for the shell process + * to complete it's execution. If at this point it is still + * alive, it (most probably) is stuck on a blocking read + * waiting for the user input. Since a few of the utilities + * performing such blocking reads don't respond to SIGINT + * (e.g. pax(1)), we terminate the shell process via SIGTERM. + */ + if (kill(pipe_descr->pid, SIGTERM) < 0) + logging::LogPerror("kill()"); + } + + /* Retrieve exit status of the shell process. */ + do { + pid = wait4(pipe_descr->pid, &pstat, 0, (struct rusage *)0); + } while (pid == -1 && errno == EINTR); + + fclose(pipe); + exitstatus = (pid == -1) ? -1 : WEXITSTATUS(pstat); + DEBUGP("Command: %s, exit status: %d\n", command.c_str(), exitstatus); + + return std::make_pair + ((std::string)usage_output, (int)exitstatus); +}