Index: tools/tools/smoketestsuite/Makefile =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/Makefile @@ -0,0 +1,31 @@ +# $FreeBSD$ +# +# Makefile for building the test generation tool + +PROG_CXX= generate_tests +LOCALBASE= /usr/local +MAN= +CXXFLAGS+= -I${LOCALBASE}/include -std=c++11 +LDFLAGS+= -L${LOCALBASE}/lib -lboost_filesystem -lboost_system +SRCS= logging.cpp \ + utils.cpp \ + read_annotations.cpp \ + generate_license.cpp \ + add_testcase.cpp \ + fetch_groff.cpp \ + generate_test.cpp + +.PHONY: clean \ + fetch_utils \ + run + +fetch_utils: + sh ${.CURDIR}/scripts/fetch_utils.sh + +run: + @echo Generating annotations... + sh ${.CURDIR}/scripts/generate_annot.sh + @echo Generating test files... + ./generate_tests + +.include Index: tools/tools/smoketestsuite/README =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/README @@ -0,0 +1,50 @@ +$FreeBSD$ + +Smoke testing of base utilities +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Test generation tool made as a part of Google Summer of Code '17 with FreeBSD (https://summerofcode.withgoogle.com/projects/#6426676740227072). +Refer the FreeBSD wiki (https://wiki.freebsd.org/SummerOfCode2017/SmokeTestingOfBaseUtilities) for an overview and updates. +A brief description and implementation details: https://lists.freebsd.org/pipermail/soc-status/2017-July/001079.html. +The diagram "architecture.png" briefly summarizes how different components fit with the testcase-generator. + +Project layout +~~~~~~~~~~~~~~ + +. +├── annotations +│   └── ........................:: Annotation files (generated/user-defined) +├── scripts +│   └── ........................:: Helper scripts +├── architecture.png ...........:: A brief architecture diagram +├── add_testcase.cpp ...........:: Testcase generator +├── generate_license.cpp .......:: Customized license generator +├── generate_test.cpp ..........:: Test generator +├── logging.cpp ................:: Logger +├── read_annotations.cpp .......:: Annotation parser +└── utils.cpp ..................:: Index generator + +- - - + +Dependencies +~~~~~~~~~~~~ +* Boost C++ libraries: The tool was tested to work with the port "boost-all-1.64.0". +* Kyua + +Instructions +~~~~~~~~~~~~ +* The tool needs to know about the utilities in src which don't already have tests. The list of all such utilities can be generated via - + + make fetch_utils + + This will generate the file "scripts/utils_list". + +* For generating the tests, execute the following - + + make clean + make && make run + +ToDo +~~~~ +The following features/functionalities are planned to be integrated - +* [Batch mode] Appropriately update `BSD.tests.dist`. Index: tools/tools/smoketestsuite/add_testcase.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/add_testcase.h @@ -0,0 +1,43 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ADD_TESTCASE_H_ +#define _ADD_TESTCASE_H_ + +namespace addtestcase { + void KnownTestcase(std::string, std::string, std::string, \ + std::string, std::ofstream&); + + void UnknownTestcase(std::string, std::string, std::pair, \ + std::string&, bool); + + void NoArgsTestcase(std::string, std::pair, \ + std::ofstream&, bool); +} + +#endif /* _ADD_TESTCASE_H_ */ Index: tools/tools/smoketestsuite/add_testcase.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/add_testcase.cpp @@ -0,0 +1,177 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include + +#include "add_testcase.h" + +/* Adds a test-case for an option with known usage. */ +void +addtestcase::KnownTestcase(std::string option, + std::string util_with_section, + std::string descr, + std::string output, + std::ofstream& test_script) +{ + std::string testcase_name; + std::string utility = util_with_section.substr(0, + util_with_section.size() - 3); + + /* Add testcase name. */ + test_script << "atf_test_case "; + if (!option.empty()) { + testcase_name = option; + testcase_name.append("_flag"); + } else + testcase_name = "no_arguments"; + test_script << testcase_name + "\n"; + + /* Add testcase description. */ + test_script << testcase_name + + "_head()\n{\n\tatf_set \"descr\" "; + if (!descr.empty()) + test_script << descr; + else + test_script << "\"Verify the usage of option \'" + + option + "\'\""; + test_script << "\n}\n\n"; + + /* Add body of the testcase. */ + test_script << testcase_name + "_body()\n{" + + "\n\tatf_check -s exit:0 -o "; + + /* Match the usage output if generated. */ + if (!output.empty()) + test_script << "inline:\"" + output + "\" "; + else + test_script << "empty "; + test_script << utility; + + if (!option.empty()) + test_script << " -" + option; + test_script << "\n}\n\n"; +} + +/* Adds a test-case for an option with unknown usage. */ +void +addtestcase::UnknownTestcase(std::string option, + std::string util_with_section, + std::pair output, + std::string& testcase_buffer, + bool usage_output) +{ + std::string utility = util_with_section.substr(0, + util_with_section.size() - 3); + + if (output.second) + testcase_buffer.append("\n\tatf_check -s not-exit:0 -e "); + else + testcase_buffer.append("\n\tatf_check -s exit:0 -o "); + + /* Check if a usage message was produced (case-insensitive match). */ + if (usage_output) + testcase_buffer.append("match:\"$usage_output\" "); + else if (!output.first.empty()) + testcase_buffer.append("inline:\"" + output.first + "\" "); + else + testcase_buffer.append("empty "); + + testcase_buffer.append(utility); + + if (!option.empty()) + testcase_buffer.append(" -" + option); +} + +/* Adds a test-case for usage without any arguments. */ +void +addtestcase::NoArgsTestcase(std::string util_with_section, + std::pair output, + std::ofstream& test_script, + bool usage_output) +{ + std::string descr; + std::string utility = util_with_section.substr(0, + util_with_section.size() - 3); + + if (output.second) { + /* An error was encountered. */ + test_script << std::string("atf_test_case no_arguments\n") + + "no_arguments_head()\n{\n\tatf_set \"descr\" "; + if (!output.first.empty()) { + /* + * We expect a usage message to be generated in this + * case (case-insensitive match). + */ + if (usage_output) { + descr = "\"Verify that " + util_with_section + + " fails and generates a valid usage \" " + + "\\\n\t\t\t\"message when no arguments " + + "are supplied\""; + + test_script << descr + + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e match:" + + "\"$usage_output\" " + utility; + } else { + descr = "\"Verify that " + util_with_section + + " fails and generates a valid output \" " + + "\\\n\t\t\t\"when no arguments are supplied\""; + + test_script << descr + + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e inline:" + + "\"" + output.first + "\" " + + utility; + } + } else { + descr = "\"Verify that " + util_with_section + " fails " + + "silently when no arguments are supplied\"" ; + test_script << descr + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e empty " + + utility; + } + test_script << "\n}\n\n"; + } else { + /* + * The command ran successfully, hence we guessed + * a correct usage for the utility under test. + */ + if (!output.first.empty()) + descr = "\"Verify that " + util_with_section + " executes " + + "successfully and produces a valid \" \\\n\t\t\t" + + "\"output when invoked without any arguments\""; + else + descr = "\"Verify that " + util_with_section + + " executes successfully and silently \" \\\n" + + "\t\t\t\"when invoked without any arguments\""; + + addtestcase::KnownTestcase("", util_with_section, descr, + output.first, test_script); + } +} Index: tools/tools/smoketestsuite/annotations/date_test.ant =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/annotations/date_test.ant @@ -0,0 +1,5 @@ +R_flag +j_flag +n_flag +no_arguments +u_flag Index: tools/tools/smoketestsuite/fetch_groff.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/fetch_groff.h @@ -0,0 +1,39 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _FETCH_GROFF_H_ +#define _FETCH_GROFF_H_ + +#include + +namespace groff { + extern std::unordered_map groff_map; + int FetchGroffScripts(); +} + +#endif /* _FETCH_GROFF_H_ */ Index: tools/tools/smoketestsuite/fetch_groff.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/fetch_groff.cpp @@ -0,0 +1,98 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "fetch_groff.h" +#include "logging.h" + +/* Map of utility name and its location in src tree. */ +std::unordered_map groff::groff_map; + +/* + * Traverses the FreeBSD src tree looking for groff scripts for section + * 1 and section 8 utilities and stores their location in a hashmap. + */ +int +groff::FetchGroffScripts() +{ + std::string utils_list = "scripts/utils_list"; + std::string src = "../../../"; /* FreeBSD src. */ + std::string utildir; + std::string utilname; + std::string path; + std::ifstream file; + std::regex section ("(.*).(?:1|8)"); + struct stat sb; + struct dirent *ent; + DIR *dir; + + /* Check if the file "scripts/utils_list" exists. */ + if (stat(utils_list.c_str(), &sb) != 0) { + std::cerr << "scripts/utils_list does not exists.\n" + "Run 'make fetch_utils' first.\n"; + return EXIT_FAILURE; + } + file.open(utils_list); + + while (getline(file, utildir)) { + path = src + utildir + "/tests"; + /* + * Copy the groff script only if the utility does not + * already have tests, i.e. the "tests" directory is absent. + */ + if (stat(path.c_str(), &sb) || !S_ISDIR(sb.st_mode)) { + path = src + utildir + "/"; + utilname = utildir.substr(utildir.find_last_of("/") + 1); + if ((dir = opendir(path.c_str())) != NULL) { + /* Skip directory entry for "." and "..". */ + ent = readdir(dir); + ent = readdir(dir); + while ((ent = readdir(dir)) != NULL) { + if (std::regex_match(ent->d_name, section)) { + groff_map[utilname] = path + ent->d_name; + } + } + closedir(dir); + } else { + logging::LogPerror("opendir()"); + } + } + } + + file.close(); + return EXIT_SUCCESS; +} Index: tools/tools/smoketestsuite/generate_license.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_license.h @@ -0,0 +1,36 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _GENERATE_LICENSE_H_ +#define _GENERATE_LICENSE_H_ + +namespace generatelicense { + std::string GenerateLicense(int, char **); +} + +#endif /* _GENERATE_LICENSE_H_ */ Index: tools/tools/smoketestsuite/generate_license.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_license.cpp @@ -0,0 +1,82 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include "generate_license.h" +#include "utils.h" + +std::string +generatelicense::GenerateLicense(int argc, char **argv) +{ + std::string license; + std::string copyright_owner; + + if (argc > 1) { + if (argc == 3 && strncmp(argv[1], "--name ", 7)) + copyright_owner = argv[2]; + else { + std::cerr << "Usage: ./generate_tests --name \n"; + exit(EXIT_FAILURE); + } + } else + copyright_owner = utils::Execute("id -P | cut -d : -f 8").first; + + license = + "#\n" + "# Copyright 2017-2018 " + copyright_owner + "\n" + "# All rights reserved.\n" + "#\n" + "# Redistribution and use in source and binary forms, with or without\n" + "# modification, are permitted provided that the following conditions\n" + "# are met:\n" + "# 1. Redistributions of source code must retain the above copyright\n" + "# notice, this list of conditions and the following disclaimer.\n" + "# 2. Redistributions in binary form must reproduce the above copyright\n" + "# notice, this list of conditions and the following disclaimer in the\n" + "# documentation and/or other materials provided with the distribution.\n" + "#\n" + "# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND\n" + "# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n" + "# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n" + "# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n" + "# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n" + "# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n" + "# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n" + "# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n" + "# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n" + "# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n" + "# SUCH DAMAGE.\n" + "#\n" + "# $FreeBSD$\n" + "#\n\n"; + + return license; +} Index: tools/tools/smoketestsuite/generate_test.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_test.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _GENERATE_TEST_H_ +#define _GENERATE_TEST_H_ + +#include "utils.h" + +namespace generatetest { + void IntHandler(int); + void GenerateMakefile(std::string, std::string); + void GenerateTest(std::string, char, + std::string&, const char*); +} + +#endif /* _GENERATE_TEST_H_ */ Index: tools/tools/smoketestsuite/generate_test.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_test.cpp @@ -0,0 +1,404 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "add_testcase.h" +#include "fetch_groff.h" +#include "generate_license.h" +#include "generate_test.h" +#include "logging.h" +#include "read_annotations.h" + +void +generatetest::IntHandler(int dummmy) +{ + std::cerr << "\nExiting...\n"; + /* Remove the temporary directory. */ + boost::filesystem::remove_all(utils::tmpdir); + exit(EXIT_FAILURE); +} + +/* [Batch mode] Generate a makefile for the test of given utility. */ +void +generatetest::GenerateMakefile(std::string utility, std::string utildir) +{ + std::ofstream file; + + file.open(utildir + "/Makefile", std::ios::out); + file << "# $FreeBSD$\n\nATF_TESTS_SH+= " + + utility + "_test\n\n" + + ".include \n"; + file.close(); +} + +/* Generate a test for the given utility. */ +void +generatetest::GenerateTest(std::string utility, + char section, + std::string& license, + const char *testsdir) +{ + std::vector usage_messages; + std::vector identified_opts; + std::string command; + std::string testcase_list; + std::string buffer; + std::string testfile; + std::string util_with_section; + std::ofstream file; + std::pair output; + std::unordered_set annotation_set; + /* Number of options for which a testcase has been generated. */ + int progress = 0; + bool usage_output = false; /* Tracks whether '$usage_output' variable is used. */ + + /* Read annotations and populate hash set "annotation_set". */ + annotations::read_annotations(utility, annotation_set); + util_with_section = utility + '(' + section + ')'; + utils::OptDefinition opt_def; + identified_opts = opt_def.CheckOpts(utility); + testfile = testsdir + utility + "_test.sh"; + +#ifndef DEBUG + /* Indicate the start of test generation for current utility. */ + if (isatty(fileno(stderr))) { + std::cerr << std::setw(18) << util_with_section << " | " + << progress << "/" << opt_def.opt_list.size() << "\r"; + } +#endif + /* Add license in the generated test scripts. */ + file.open(testfile, std::ios::out); + file << license; + + /* + * If a known option was encountered (i.e. `identified_opts` is + * populated), produce a testcase to check the validity of the + * result of that option. If no known option was encountered, + * produce testcases to verify the correct (generated) usage + * message when using the supported options incorrectly. + */ + for (const auto &i : identified_opts) { + command = utils::GenerateCommand(utility, i->value); + output = utils::Execute(command); + if (boost::iequals(output.first.substr(0, 6), "usage:")) { + /* Our guessed usage is incorrect as usage message is produced. */ + addtestcase::UnknownTestcase(i->value, util_with_section, + output, buffer, usage_output); + } else { + addtestcase::KnownTestcase(i->value, util_with_section, + "", output.first, file); + } + testcase_list.append("\tatf_add_test_case " + i->value + "_flag\n"); + } + + /* Add testcases for the options whose usage is not yet known. + * For the purpose of adding a "$usage_output" variable, + * we choose the option which produces one. + * TODO Avoid double executions of an option, i.e. one while + * selecting usage message and another while generating testcase. + */ + if (opt_def.opt_list.size() == 1) { + /* Check if the single option produces a usage message. */ + command = utils::GenerateCommand(utility, opt_def.opt_list.front()); + output = utils::Execute(command); + if (output.second && !output.first.empty()) { + usage_output = true; + file << "usage_output=\'" + output.first + "\'\n\n"; + } + } else if (opt_def.opt_list.size() > 1) { + /* + * Utility supports multiple options. In case the usage message + * is consistent for atleast "two" options, we reduce duplication + * by assigning a variable "usage_output" in the test script. + */ + for (const auto &i : opt_def.opt_list) { + command = utils::GenerateCommand(utility, i); + output = utils::Execute(command); + if (output.second && usage_messages.size() < 3) + usage_messages.push_back(output.first); + } + + for (int j = 0; j < usage_messages.size(); j++) { + if (!usage_messages[j].compare + (usage_messages[(j+1) % usage_messages.size()])) { + usage_output = true; + file << "usage_output=\'" + + output.first.substr(0, 7 + utility.size()) + + "\'\n\n"; + break; + } + } + } + + /* + * Execute the utility with supported options, while + * adding positive and negative testcases accordingly. + */ + for (const auto &i : opt_def.opt_list) { + /* Ignore the option if it is annotated. */ + if (annotation_set.find(i) != annotation_set.end()) + continue; + + command = utils::GenerateCommand(utility, i); + output = utils::Execute(command); +#ifndef DEBUG + if (isatty(fileno(stderr))) { + std::cerr << std::setw(18) << util_with_section + << " | " << ++progress << "/" + << opt_def.opt_list.size() << "\r"; + } +#endif + if (output.second) { + addtestcase::UnknownTestcase(i, util_with_section, output, + buffer, usage_output); + } else { + /* Guessed usage is correct as EXIT_SUCCESS is encountered */ + addtestcase::KnownTestcase(i, util_with_section, "", + output.first, file); + testcase_list.append(std::string("\tatf_add_test_case ") + + i + "_flag\n"); + } + } + std::cout << std::endl; /* Takes care of the last '\r'. */ + + if (!opt_def.opt_list.empty()) { + testcase_list.append("\tatf_add_test_case invalid_usage\n"); + file << "atf_test_case invalid_usage\ninvalid_usage_head()\n" + << "{\n\tatf_set \"descr\" \"Verify that an invalid usage " + << "with a supported option \" \\\n\t\t\t\"produces a valid " + << "error message\"\n}\n\ninvalid_usage_body()\n{"; + + file << buffer + "\n}\n\n"; + } + + /* + * Add a testcase under "no_arguments" for + * running the utility without any arguments. + */ + if (annotation_set.find("*") == annotation_set.end()) { + command = utils::GenerateCommand(utility, ""); + output = utils::Execute(command); + addtestcase::NoArgsTestcase(util_with_section, output, + file, usage_output); + testcase_list.append("\tatf_add_test_case no_arguments\n"); + } + + file << "atf_init_test_cases()\n{\n" + testcase_list + "}\n"; + file.close(); +} + +int +main(int argc, char **argv) +{ + std::ifstream groff_list; + struct stat sb; + struct dirent *ent; + char answer; + std::string license; + std::string utildir; /* Path to utility in src tree. */ + std::string groffpath; + const char *testsdir = "generated_tests/"; + /* + * Instead of generating tests for all the utilities, "batch mode" + * allows generation of tests for first "batch_limit" number of + * utilities selected from "scripts/utils_list". + */ + bool batch_mode = false; + int batch_limit; /* Number of tests to be generated in batch mode. */ + + /* Handle interrupts. */ + signal(SIGINT, generatetest::IntHandler); + + if (groff::FetchGroffScripts() == EXIT_FAILURE) + return EXIT_FAILURE; + + /* + * Create a temporary directory where all the side-effects + * introduced by utility-specific commands are restricted. + */ + boost::filesystem::create_directory(utils::tmpdir); + + std::cout << "\nInstead of generating tests for all the utilities, 'batch mode'\n" + "allows generation of tests for first few utilities selected from\n" + "'scripts/utils_list', and places them at their correct location\n" + "in the src tree, with corresponding makefiles created.\n" + "NOTE: You will be prompted for the superuser password when\n" + "creating test directory under '/usr/tests/' and when installing\n" + "the tests via `sudo make install`.\n" + "Run in 'batch mode' ? [y/N] "; + std::cin.get(answer); + + switch(answer) { + case 'y': + case 'Y': + batch_mode = true; + std::cout << "Number of utilities to select for test generation: "; + std::cin >> batch_limit; + + if (batch_limit <= 0) { + std::cerr << "Invalid input. Exiting...\n"; + return EXIT_FAILURE; + } + break; + case '\n': + default: + break; + } + + /* Check if the directory "testsdir" exists. */ + if (stat(testsdir, &sb) || !S_ISDIR(sb.st_mode)) { + boost::filesystem::path dir(testsdir); + if (boost::filesystem::create_directory(dir)) + std::cout << "Directory created: " << testsdir << "\n"; + else { + std::cerr << "Unable to create directory: " << testsdir << "\n"; + return EXIT_FAILURE; + } + } + + /* Generate a license to be added in the generated scripts. */ + license = generatelicense::GenerateLicense(argc, argv); + +#ifndef DEBUG + /* Generate a tabular-like format. */ + std::cout << std::endl; + std::cout << std::setw(30) << "Utility | Progress\n"; + std::cout << std::setw(32) << "----------+-----------\n"; +#endif + + if (batch_mode) { + /* Number of hops required to reach root directory. */ + std::string hops = "../../../../"; + std::string command; + std::string installdir; /* Directory where tests are installed. */ + int retval; + int maxdepth = 32; + std::unordered_map::iterator it; + /* Remember pwd so that we can return back. */ + boost::filesystem::path tooldir = boost::filesystem::current_path(); + + /* + * Discover number of hops required to reach root directory. + * To avoid an infinite loop in case directory "usr/" doesn't + * exist in the filesystem, we assume that the maximum depth + * from root directory at which pwd is located is "maxdepth". + */ + chdir(hops.c_str()); /* Move outside FreeBSD src. */ + while (maxdepth--) { + if (chdir("..") == -1) { + perror("chdir"); + return EXIT_FAILURE; + } + hops += "../"; + if (stat("usr", &sb) == 0 && S_ISDIR(sb.st_mode)) + break; + } + + /* + * Generate tests for first "batch_limit" number of + * utilities selected from "scripts/utils_list". + */ + it = groff::groff_map.begin(); + while (batch_limit-- && it != groff::groff_map.end()) { + /* Move back to the tool's directory. */ + boost::filesystem::current_path(tooldir); + + groffpath = groff::groff_map.at(it->first); + utildir = groffpath.substr + (0, groffpath.size() - 2 - it->first.size()); + installdir = hops + "usr/tests/" + utildir.substr(9); + utildir += "tests/"; + + /* Populate "tests/" directory. */ + boost::filesystem::remove_all(utildir); + boost::filesystem::create_directory(utildir); + generatetest::GenerateMakefile(it->first, utildir); + generatetest::GenerateTest(it->first, it->second.back(), + license, utildir.c_str()); + boost::filesystem::copy_file(utildir + it->first + "_test.sh", + testsdir + it->first + "_test.sh", + boost::filesystem::copy_option::overwrite_if_exists); + std::advance(it, 1); + + /* Execute the generated test and note success/failure. */ + if (stat(installdir.c_str(), &sb) || !S_ISDIR(sb.st_mode)) { + command = "sudo mkdir -p " + installdir; + if ((retval = system(command.c_str())) == -1) { + perror("system"); + return EXIT_FAILURE; + } else if (retval) { + boost::filesystem::current_path(tooldir); + boost::filesystem::remove_all(utildir); + continue; + } + } + + /* Install the test. */ + chdir(utildir.c_str()); + if ((retval = system("sudo make install")) == -1) { + perror("system"); + return EXIT_FAILURE; + } else if (retval) { + boost::filesystem::current_path(tooldir); + boost::filesystem::remove_all(utildir); + continue; + } + boost::filesystem::current_path(tooldir); + + /* Run the test. */ + chdir(installdir.c_str()); + if ((retval = system("kyua test")) == -1) { + perror("system"); + return EXIT_FAILURE; + } else if (retval) { + boost::filesystem::current_path(tooldir); + boost::filesystem::remove_all(utildir); + continue; + } + } + } else { + for (const auto &it : groff::groff_map) { + generatetest::GenerateTest(it.first, it.second.back(), + license, testsdir); + } + } + + /* Cleanup. */ + boost::filesystem::remove_all(utils::tmpdir); + return EXIT_SUCCESS; +} Index: tools/tools/smoketestsuite/logging.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/logging.h @@ -0,0 +1,45 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LOGGING_H_ +#define _LOGGING_H_ + +/* Use a gcc variadic macro to conditionally compile debug printing. */ +#ifdef DEBUG +#define DEBUGP(...) \ + fprintf(stdout, __VA_ARGS__); \ + fflush(stdout); +#else +#define DEBUGP(...) {} +#endif /* DEBUG */ + +namespace logging { + void LogPerror(const char *); +} + +#endif /* _LOGGING_H_ */ Index: tools/tools/smoketestsuite/logging.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/logging.cpp @@ -0,0 +1,39 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include + +#include "logging.h" + +void +logging::LogPerror(const char *message) +{ +#ifdef DEBUG + perror(message); +#endif +} Index: tools/tools/smoketestsuite/read_annotations.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/read_annotations.h @@ -0,0 +1,39 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _READ_ANNOTATIONS_H_ +#define _READ_ANNOTATIONS_H_ + +#include + +namespace annotations { + void read_annotations(std::string, \ + std::unordered_set&); +} + +#endif /* _READ_ANNOTATIONS_H_ */ Index: tools/tools/smoketestsuite/read_annotations.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/read_annotations.cpp @@ -0,0 +1,58 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include "read_annotations.h" + +/* Read the annotation files and skip generation of respective tests. */ +void +annotations::read_annotations(std::string utility, + std::unordered_set& annotation_set) +{ + std::string line; + std::ifstream file; + file.open("annotations/" + utility + "_test.annot"); + + while (getline(file, line)) { + /* Add a unique identifier for no_arguments testcase */ + if (!line.compare(0, 12, "no_arguments")) + annotation_set.insert("*"); + /* + * Add flag value for supported argument testcases + * Doing so we ignore the "invalid_usage" testcase + * as it is guaranteed to always succeed. + */ + else if (!line.compare(2, 4, "flag")) + annotation_set.insert(line.substr(0, 1)); + } + + file.close(); +} Index: tools/tools/smoketestsuite/scripts/README =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/README @@ -0,0 +1,9 @@ +Helper scripts +~~~~~~~~~~~~~~ + +Script Name | Functionality +------------------+----------------- +fetch_groff.sh | Lists location of all section 1 utilities +fetch_utils.sh | Saves all the base utilities in the src tree in **utils_list** +generate_annot.sh | Populates annotation files under [annotations](../annotations) +validate.sh | Validates side-effects of newly introduced changes in the tool Index: tools/tools/smoketestsuite/scripts/fetch_utils.sh =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/fetch_utils.sh @@ -0,0 +1,44 @@ +#! /bin/sh +# +# Copyright 2017-2018 Shivansh Rai +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +# Script for listing all the base utilities + +set -eu + +script_dir="$(dirname $0)" +src="$(dirname $0)/../../../../" + +fetch_utils() { + cd "$src" + find . -name Makefile | xargs grep -l 'PROG\|PROG_CXX' \ + | sed -e 's|/Makefile$||' | cut -c 3- +} + +rm -f "$script_dir/utils_list" + +(fetch_utils) >> "$script_dir/utils_list" Index: tools/tools/smoketestsuite/scripts/generate_annot.sh =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/generate_annot.sh @@ -0,0 +1,107 @@ +#!/bin/sh +# +# Copyright 2017-2018 Shivansh Rai +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +# Script for generating annotations based on generated tests. + +set -eu + +update_annotations() { + cd "$testdir" 2>/dev/null + report=$(kyua report) + cd - + i=2 + annotations="" + diff="" + + while true + do + testcase=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $1}') + exitstatus=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $3}') + check=$(printf "%s" "$testcase" | cut -s -f1 -d":") + + if [ ! "$check" ] && [ "$annotations" ]; then + file="$tooldir/annotations/$test.ant" # Annotations file + # Append only the new annotations. + printf "$annotations" > "$file.temp" + [ ! -e "$file" ] && touch "$file" + diff=$(comm -13 "$file" "$file.temp") + if [ "$diff" ]; then + if [ $prompt = 0 ]; then + prompt=1 + printf "\nModified annotation files -\n" + fi + printf "$diff\n" >> "$file" + printf " * %s\n" "$test.ant" + fi + rm -f "$file.temp" + break + fi + + if [ "$exitstatus" = "failed:" ]; then + testcase=${testcase#"$test:"} + annotations="$annotations$testcase\n" + elif [ ! "$exitstatus" ]; then + break + fi + i=$((i+1)) + done +} + +suffix="_test" +extension=".sh" +prompt=0 +tooldir=$(dirname $0)/.. + +# Check if the directory "generated_tests/" is populated. +if [ ! -d "$tooldir/generated_tests" ] || \ + [ -z "$(ls -A $tooldir/generated_tests)" ]; then + exit +fi + +for f in "$tooldir/generated_tests"/* +do + file=$(basename "$f") + test=${file%$extension} + utility=${test%$suffix} + + # Guess the correct installation directory. + if [ -d "/usr/tests/bin/$utility" ]; then + testdir="/usr/tests/bin/$utility" + elif [ -d "/usr/tests/usr.bin/$utility" ]; then + testdir="/usr/tests/usr.bin/$utility" + elif [ -d "/usr/tests/usr.sbin/$utility" ]; then + testdir="/usr/tests/usr.sbin/$utility" + else + continue + fi + update_annotations +done + +if [ $prompt = 1 ]; then + printf "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n" +fi Index: tools/tools/smoketestsuite/scripts/utils_list =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/utils_list @@ -0,0 +1,1251 @@ + +bin/cat +bin/chflags +bin/chio +bin/chmod +bin/cp +bin/csh +bin/date +bin/dd +bin/df +bin/domainname +bin/echo +bin/ed +bin/expr +bin/getfacl +bin/hostname +bin/kenv +bin/kill +bin/ln +bin/ls +bin/mkdir +bin/mv +bin/pax +bin/pkill +bin/ps +bin/pwait +bin/pwd +bin/rcp +bin/realpath +bin/rm +bin/rmail +bin/rmdir +bin/setfacl +bin/sh +bin/sleep +bin/stty +bin/sync +bin/test +bin/uuidgen +cddl/sbin/zfs +cddl/sbin/zpool +cddl/usr.bin/ctfconvert +cddl/usr.bin/ctfdump +cddl/usr.bin/ctfmerge +cddl/usr.bin/zinject +cddl/usr.bin/zlook +cddl/usr.bin/zstreamdump +cddl/usr.bin/ztest +cddl/usr.sbin/dtrace +cddl/usr.sbin/lockstat +cddl/usr.sbin/plockstat +cddl/usr.sbin/zdb +cddl/usr.sbin/zfsd +cddl/usr.sbin/zhack +contrib/blacklist/bin +contrib/blacklist/test +contrib/bmake +contrib/elftoolchain/addr2line +contrib/elftoolchain/ar +contrib/elftoolchain/brandelf +contrib/elftoolchain/cxxfilt +contrib/elftoolchain/elfcopy +contrib/elftoolchain/elfdump +contrib/elftoolchain/nm +contrib/elftoolchain/readelf +contrib/elftoolchain/size +contrib/elftoolchain/strings +contrib/zstd/contrib/pzstd +contrib/zstd/programs +contrib/zstd/zlibWrapper +contrib/ldns-host +contrib/mdocml +contrib/mtree +contrib/ofed/libibverbs/examples/build/asyncwatch +contrib/ofed/libibverbs/examples/build/device_list +contrib/ofed/libibverbs/examples/build/devinfo +contrib/ofed/libibverbs/examples/build/rc_pingpong +contrib/ofed/libibverbs/examples/build/srq_pingpong +contrib/ofed/libibverbs/examples/build/uc_pingpong +contrib/ofed/libibverbs/examples/build/ud_pingpong +contrib/ofed/librdmacm/examples/build/cmatose +contrib/ofed/librdmacm/examples/build/mckey +contrib/ofed/librdmacm/examples/build/rping +contrib/ofed/librdmacm/examples/build/udaddy +contrib/ofed/usr.bin/ibaddr +contrib/ofed/usr.bin/ibnetdiscover +contrib/ofed/usr.bin/ibping +contrib/ofed/usr.bin/ibportstate +contrib/ofed/usr.bin/ibroute +contrib/ofed/usr.bin/ibsendtrap +contrib/ofed/usr.bin/ibstat +contrib/ofed/usr.bin/ibsysstat +contrib/ofed/usr.bin/ibtracert +contrib/ofed/usr.bin/opensm +contrib/ofed/usr.bin/osmtest +contrib/ofed/usr.bin/perfquery +contrib/ofed/usr.bin/saquery +contrib/ofed/usr.bin/sminfo +contrib/ofed/usr.bin/smpdump +contrib/ofed/usr.bin/smpquery +contrib/ofed/usr.bin/vendstat +contrib/pnpinfo +contrib/smbfs/smbutil +contrib/tzcode/zic/zdump +contrib/tzcode/zic/zic +contrib/unvis +contrib/vis +crypto/openssh/regress/misc/kexfuzz +crypto/openssh/regress/unittests/bitmap +crypto/openssh/regress/unittests/hostkeys +crypto/openssh/regress/unittests/kex +crypto/openssh/regress/unittests/match +crypto/openssh/regress/unittests/sshbuf +crypto/openssh/regress/unittests/sshkey +crypto/openssh/regress/unittests/utf8 +crypto/openssh/regress/unittests/conversion +crypto/openssl +crypto/openssl/apps +crypto/openssl/crypto +crypto/openssl/crypto/aes +crypto/openssl/crypto/asn1 +crypto/openssl/crypto/bf +crypto/openssl/crypto/bio +crypto/openssl/crypto/bn +crypto/openssl/crypto/buffer +crypto/openssl/crypto/camellia +crypto/openssl/crypto/cast +crypto/openssl/crypto/cmac +crypto/openssl/crypto/cms +crypto/openssl/crypto/des +crypto/openssl/crypto/dh +crypto/openssl/crypto/dsa +crypto/openssl/crypto/dso +crypto/openssl/crypto/ec +crypto/openssl/crypto/ecdh +crypto/openssl/crypto/ecdsa +crypto/openssl/crypto/engine +crypto/openssl/crypto/err +crypto/openssl/crypto/hmac +crypto/openssl/crypto/idea +crypto/openssl/crypto/jpake +crypto/openssl/crypto/lhash +crypto/openssl/crypto/md2 +crypto/openssl/crypto/md4 +crypto/openssl/crypto/md5 +crypto/openssl/crypto/mdc2 +crypto/openssl/crypto/modes +crypto/openssl/crypto/objects +crypto/openssl/crypto/pkcs12 +crypto/openssl/crypto/pkcs7 +crypto/openssl/crypto/pqueue +crypto/openssl/crypto/rand +crypto/openssl/crypto/rc2 +crypto/openssl/crypto/rc4 +crypto/openssl/crypto/rc5 +crypto/openssl/crypto/ripemd +crypto/openssl/crypto/rsa +crypto/openssl/crypto/seed +crypto/openssl/crypto/sha +crypto/openssl/crypto/srp +crypto/openssl/crypto/stack +crypto/openssl/crypto/store +crypto/openssl/crypto/ts +crypto/openssl/crypto/txt_db +crypto/openssl/crypto/ui +crypto/openssl/crypto/whrlpool +crypto/openssl/crypto/x509 +crypto/openssl/crypto/x509v3 +crypto/openssl/engines +crypto/openssl/engines/ccgost +crypto/openssl/ssl +gnu/usr.bin/binutils/as +gnu/usr.bin/binutils/ld +gnu/usr.bin/binutils/objcopy +gnu/usr.bin/binutils/objdump +gnu/usr.bin/cc/c++ +gnu/usr.bin/cc/cc +gnu/usr.bin/cc/cc1 +gnu/usr.bin/cc/cc1plus +gnu/usr.bin/cc/cc_tools +gnu/usr.bin/cc/cpp +gnu/usr.bin/cc/gcov +gnu/usr.bin/dialog +gnu/usr.bin/diff3 +gnu/usr.bin/dtc +gnu/usr.bin/gdb/gdb +gnu/usr.bin/gdb/gdbserver +gnu/usr.bin/gdb/gdbtui +gnu/usr.bin/gdb/kgdb +gnu/usr.bin/gperf +gnu/usr.bin/grep +kerberos5 +kerberos5/libexec/digest-service +kerberos5/libexec/hprop +kerberos5/libexec/hpropd +kerberos5/libexec/ipropd-master +kerberos5/libexec/ipropd-slave +kerberos5/libexec/kadmind +kerberos5/libexec/kcm +kerberos5/libexec/kdc +kerberos5/libexec/kdigest +kerberos5/libexec/kfd +kerberos5/libexec/kimpersonate +kerberos5/libexec/kpasswdd +kerberos5/tools/asn1_compile +kerberos5/tools/make-roken +kerberos5/tools/slc +kerberos5/usr.bin/hxtool +kerberos5/usr.bin/kadmin +kerberos5/usr.bin/kcc +kerberos5/usr.bin/kdestroy +kerberos5/usr.bin/kf +kerberos5/usr.bin/kgetcred +kerberos5/usr.bin/kinit +kerberos5/usr.bin/kpasswd +kerberos5/usr.bin/ksu +kerberos5/usr.bin/string2key +kerberos5/usr.bin/verify_krb5_conf +kerberos5/usr.sbin/iprop-log +kerberos5/usr.sbin/kstash +kerberos5/usr.sbin/ktutil +lib/atf/libatf-c++/tests/detail +lib/atf/libatf-c/tests/detail +lib/libarchive/tests +lib/libblacklist +lib/libc/db/test +lib/libc/tests/db +lib/libc/tests/gen +lib/libc/tests/gen/posix_spawn +lib/libc/tests/hash +lib/libc/tests/net +lib/libc/tests/net/getaddrinfo +lib/libc/tests/regex +lib/libc/tests/ssp +lib/libc/tests/stdio +lib/libc/tests/stdlib +lib/libproc/tests +lib/libthr/tests +lib/libxo/tests +lib/msun/tests +lib/ncurses/ncurses +libexec/atf/atf-check +libexec/atf/atf-sh +libexec/atrun +libexec/bootpd +libexec/bootpd/bootpgw +libexec/bootpd/tools/bootpef +libexec/bootpd/tools/bootptest +libexec/comsat +libexec/dma/dma-mbox-create +libexec/dma/dmagent +libexec/fingerd +libexec/ftpd +libexec/getty +libexec/mail.local +libexec/mknetid +libexec/pppoed +libexec/rbootd +libexec/revnetgroup +libexec/rlogind +libexec/rpc.rquotad +libexec/rpc.rstatd +libexec/rpc.rusersd +libexec/rpc.rwalld +libexec/rpc.sprayd +libexec/rshd +libexec/rtld-elf +libexec/rtld-elf/tests/target +libexec/smrsh +libexec/talkd +libexec/tcpd +libexec/telnetd +libexec/tftp-proxy +libexec/tftpd +libexec/ulog-helper +libexec/ypxfr +release/picobsd/tinyware/aps +release/picobsd/tinyware/help +release/picobsd/tinyware/login +release/picobsd/tinyware/msg +release/picobsd/tinyware/msh +release/picobsd/tinyware/ns +release/picobsd/tinyware/oinit +release/picobsd/tinyware/passwd +release/picobsd/tinyware/simple_httpd +release/picobsd/tinyware/sps +release/picobsd/tinyware/view +release/picobsd/tinyware/vm +rescue/rescue +sbin/adjkerntz +sbin/badsect +sbin/bsdlabel +sbin/camcontrol +sbin/ccdconfig +sbin/clri +sbin/comcontrol +sbin/conscontrol +sbin/ddb +sbin/decryptcore +sbin/devd +sbin/devfs +sbin/dhclient +sbin/dmesg +sbin/dump +sbin/dumpfs +sbin/dumpon +sbin/fdisk +sbin/etherswitchcfg +sbin/ffsinfo +sbin/fsck +sbin/fsck_ffs +sbin/fsck_msdosfs +sbin/fsdb +sbin/fsirand +sbin/gbde +sbin/geom +sbin/geom/core +sbin/ggate/ggatec +sbin/ggate/ggated +sbin/ggate/ggatel +sbin/growfs +sbin/gvinum +sbin/hastctl +sbin/hastd +sbin/ifconfig +sbin/init +sbin/ipf/ipf +sbin/ipf/ipfs +sbin/ipf/ipfstat +sbin/ipf/ipftest +sbin/ipf/ipmon +sbin/ipf/ipnat +sbin/ipf/ippool +sbin/ipf/ipresend +sbin/ipfw +sbin/iscontrol +sbin/kldconfig +sbin/kldload +sbin/kldstat +sbin/kldunload +sbin/ldconfig +sbin/md5 +sbin/mdconfig +sbin/mdmfs +sbin/mknod +sbin/mksnap_ffs +sbin/mount +sbin/mount_cd9660 +sbin/mount_fusefs +sbin/mount_msdosfs +sbin/mount_nfs +sbin/mount_nullfs +sbin/mount_udf +sbin/mount_unionfs +sbin/nandfs +sbin/natd +sbin/newfs +sbin/newfs_msdos +sbin/newfs_nandfs +sbin/nfsiod +sbin/nos-tun +sbin/nvmecontrol +sbin/pfctl +sbin/pflogd +sbin/ping +sbin/ping6 +sbin/quotacheck +sbin/rcorder +sbin/reboot +sbin/recoverdisk +sbin/restore +sbin/route +sbin/routed +sbin/routed/rtquery +sbin/rtsol +sbin/savecore +sbin/sconfig +sbin/setkey +sbin/shutdown +sbin/spppcontrol +sbin/sunlabel +sbin/swapon +sbin/sysctl +sbin/tunefs +sbin/umount +sbin/zfsbootcfg +secure +secure/libexec/sftp-server +secure/libexec/ssh-keysign +secure/libexec/ssh-pkcs11-helper +secure/usr.bin/openssl +secure/usr.bin/scp +secure/usr.bin/sftp +secure/usr.bin/ssh-add +secure/usr.bin/ssh-agent +secure/usr.bin/ssh-keygen +secure/usr.bin/ssh-keyscan +secure/usr.bin/ssh +secure/usr.sbin/sshd +share/examples/FreeBSD_version +share/examples/find_interface +share/examples/hwpmc +share/examples/kld/cdev/test +share/examples/kld/syscall/test +share/examples/libvgl +share/examples/perfmon +share/examples/ppi +share/examples/scsi_target +share/examples/ses/getencstat +share/examples/ses/sesd +share/examples/ses/setencstat +share/examples/ses/setobjstat +sys/boot/efi/boot1 +sys/boot/efi/loader +sys/boot/ficl +sys/boot/i386/boot0 +sys/boot/i386/boot0sio +sys/boot/i386/btx/btx +sys/boot/i386/btx/btxldr +sys/boot/i386/btx/lib +sys/boot/i386/cdboot +sys/boot/i386/gptboot +sys/boot/i386/gptzfsboot +sys/boot/i386/kgzldr +sys/boot/i386/loader +sys/boot/i386/mbr +sys/boot/i386/pmbr +sys/boot/i386/pxeldr +sys/boot/i386/zfsboot +sys/boot/mips/beri/loader +sys/boot/powerpc/boot1.chrp +sys/boot/powerpc/kboot +sys/boot/powerpc/ofw +sys/boot/powerpc/ps3 +sys/boot/powerpc/uboot +sys/boot/sparc64/boot1 +sys/boot/sparc64/loader +sys/boot/sparc64/zfsboot +sys/boot/sparc64/zfsloader +sys/boot/usb +sys/boot/usb/tools +sys/boot/userboot/ficl +sys/boot/userboot/test +sys/boot/zfs +sys/contrib/xz-embedded/userspace +sys/crypto/rijndael +sys/dev/aic7xxx/aicasm +sys/modules/aic7xxx/aicasm +sys/teken/demo +sys/teken/stress +tests/sys/acl +tests/sys/file +tests/sys/fs/tmpfs +tests/sys/kern +tests/sys/kern/execve +tests/sys/kqueue/libkqueue +tests/sys/mqueue +tests/sys/netinet +tests/sys/pjdfstest/pjdfstest +tools/KSE/ksetest +tools/KSE/rr +tools/bsdbox +tools/diag/dumpvfscache +tools/diag/localeck +tools/regression/aio/aiop +tools/regression/audit/audit_pipe_ioctl +tools/regression/doat +tools/regression/environ +tools/regression/ethernet/ethermulti +tools/regression/fsx +tools/regression/gaithrstress +tools/regression/geom/ConfCmp +tools/regression/geom/MdLoad +tools/regression/geom_gpt +tools/regression/include/stdatomic +tools/regression/include/tgmath +tools/regression/kgssapi +tools/regression/mlock +tools/regression/netinet/arphold +tools/regression/netinet/ipbroadcast +tools/regression/netinet/ipdivert +tools/regression/netinet/ipmulticast +tools/regression/netinet/ipsockopt +tools/regression/netinet/msocket +tools/regression/netinet/msocket_ifnet_remove +tools/regression/netinet/rawconnect +tools/regression/netinet/tcpconnect +tools/regression/netinet/tcpdrop +tools/regression/netinet/tcpfullwindowrst +tools/regression/netinet/tcpsockclosebeforeaccept +tools/regression/netinet/tcpsocktimewait +tools/regression/netinet/tcpstream +tools/regression/netinet/udpconnectjail +tools/regression/netinet/udpzerobyte +tools/regression/netinet6/icmp6_filter +tools/regression/netinet6/inet6_rth +tools/regression/netinet6/ip6_sockets +tools/regression/nfsmmap/test1 +tools/regression/nfsmmap/test2 +tools/regression/p1003_1b +tools/regression/poll +tools/regression/posixsem +tools/regression/posixsem2 +tools/regression/priv +tools/regression/pthread/cv_cancel1 +tools/regression/pthread/mutex_isowned_np +tools/regression/rpcsec_gss +tools/regression/security/access +tools/regression/security/cap_test +tools/regression/security/open_to_operation +tools/regression/security/proc_to_proc +tools/regression/sigqueue/sigqtest1 +tools/regression/sigqueue/sigqtest2 +tools/regression/sockets/accept_fd_leak +tools/regression/sockets/accf_data_attach +tools/regression/sockets/fstat +tools/regression/sockets/kqueue +tools/regression/sockets/listen_backlog +tools/regression/sockets/listen_kqueue +tools/regression/sockets/listenclose +tools/regression/sockets/pr_atomic +tools/regression/sockets/reconnect +tools/regression/sockets/rtsocket +tools/regression/sockets/sblock +tools/regression/sockets/sendfile +tools/regression/sockets/shutdown +tools/regression/sockets/sigpipe +tools/regression/sockets/so_setfib +tools/regression/sockets/socketpair +tools/regression/sockets/udp_pingpong +tools/regression/sockets/unix_bindconnect +tools/regression/sockets/unix_close_race +tools/regression/sockets/unix_cmsg +tools/regression/sockets/unix_gc +tools/regression/sockets/unix_sendtorace +tools/regression/sockets/unix_socket +tools/regression/sockets/unix_sorflush +tools/regression/sockets/zerosend +tools/regression/sysvmsg +tools/regression/sysvsem +tools/regression/sysvshm +tools/regression/tls/ttls1 +tools/regression/tls/ttls2 +tools/regression/tls/ttls4 +tools/regression/tmpfs +tools/regression/ufs/uprintf +tools/regression/usr.bin/env +tools/test/auxinfo +tools/test/iconv/gnu +tools/test/iconv/posix +tools/test/iconv/refgen +tools/test/iconv/tablegen +tools/test/malloc +tools/test/net +tools/test/netfibs +tools/test/ppsapi +tools/test/ptrace +tools/test/upsdl +tools/tools/aac +tools/tools/ath/arcode +tools/tools/ath/ath_ee_9287_print +tools/tools/ath/ath_ee_9300_print +tools/tools/ath/ath_ee_v14_print +tools/tools/ath/ath_ee_v4k_print +tools/tools/ath/ath_prom_read +tools/tools/ath/athaggrstats +tools/tools/ath/athalq +tools/tools/ath/athdebug +tools/tools/ath/athdecode +tools/tools/ath/athkey +tools/tools/ath/athpoke +tools/tools/ath/athpow +tools/tools/ath/athprom +tools/tools/ath/athradar +tools/tools/ath/athratestats +tools/tools/ath/athrd +tools/tools/ath/athregs +tools/tools/ath/athspectral +tools/tools/ath/athstats +tools/tools/ath/athsurvey +tools/tools/atsectl +tools/tools/bootparttest +tools/tools/bus_autoconf +tools/tools/cfi +tools/tools/cxgbtool +tools/tools/decioctl +tools/tools/dmardump +tools/tools/drm/radeon/mkregtable +tools/tools/ether_reflect +tools/tools/find-sb +tools/tools/fixwhite +tools/tools/gdb_regofs +tools/tools/gensnmpdef +tools/tools/ifinfo +tools/tools/ifpifa +tools/tools/indent_wrapper +tools/tools/ioat +tools/tools/ipw +tools/tools/iwi +tools/tools/iwn/iwnstats +tools/tools/kttcp +tools/tools/mcgrab +tools/tools/mctest +tools/tools/mfi +tools/tools/mwl/mwldebug +tools/tools/mwl/mwlstats +tools/tools/ncpus +tools/tools/net80211/stumbler +tools/tools/net80211/w00t/ap +tools/tools/net80211/w00t/assoc +tools/tools/net80211/w00t/expand +tools/tools/net80211/w00t/prga +tools/tools/net80211/w00t/redir +tools/tools/net80211/wesside/dics +tools/tools/net80211/wesside/udps +tools/tools/net80211/wesside/wesside +tools/tools/net80211/wlaninject +tools/tools/net80211/wlanstats +tools/tools/net80211/wlantxtime +tools/tools/net80211/wlanwatch +tools/tools/net80211/wlanwds +tools/tools/netmap +tools/tools/netrate/http +tools/tools/netrate/httpd +tools/tools/netrate/juggle +tools/tools/netrate/netblast +tools/tools/netrate/netreceive +tools/tools/netrate/netsend +tools/tools/netrate/tcpconnect +tools/tools/netrate/tcpp +tools/tools/netrate/tcpreceive +tools/tools/npe/npestats +tools/tools/nxge +tools/tools/pciroms +tools/tools/pirtool +tools/tools/qrndtest +tools/tools/syscall_timing +tools/tools/tionxcl +tools/tools/tscdrift +tools/tools/umastat +tools/tools/usbtest +tools/tools/vimage +tools/tools/vt/mkkfont +tools/tools/vxge +tools/tools/wtap/vis_map +tools/tools/wtap/wtap +tools/tools/zfsboottest +tools/tools/smoketestsuite +usr.bin/addr2line +usr.bin/apply +usr.bin/ar +usr.bin/asa +usr.bin/at +usr.bin/atm/sscop +usr.bin/awk +usr.bin/banner +usr.bin/basename +usr.bin/bc +usr.bin/biff +usr.bin/bluetooth/bthost +usr.bin/bluetooth/btsockstat +usr.bin/bluetooth/rfcomm_sppd +usr.bin/bmake +usr.bin/brandelf +usr.bin/bsdcat +usr.bin/bsdcat/tests +usr.bin/bsdiff/bsdiff +usr.bin/bsdiff/bspatch +usr.bin/bzip2 +usr.bin/bzip2recover +usr.bin/c89 +usr.bin/c99 +usr.bin/caesar +usr.bin/calendar +usr.bin/cap_mkdb +usr.bin/rlogin +usr.bin/chat +usr.bin/chkey +usr.bin/chpass +usr.bin/cksum +usr.bin/clang/bugpoint +usr.bin/clang/clang-format +usr.bin/clang/clang-tblgen +usr.bin/clang/clang +usr.bin/clang/llc +usr.bin/clang/lld +usr.bin/clang/lldb +usr.bin/clang/lli +usr.bin/clang/llvm-ar +usr.bin/clang/llvm-as +usr.bin/clang/llvm-bcanalyzer +usr.bin/clang/llvm-cov +usr.bin/clang/llvm-cxxdump +usr.bin/clang/llvm-cxxfilt +usr.bin/clang/llvm-diff +usr.bin/clang/llvm-dis +usr.bin/clang/llvm-dwarfdump +usr.bin/clang/llvm-extract +usr.bin/clang/llvm-link +usr.bin/clang/llvm-lto +usr.bin/clang/llvm-lto2 +usr.bin/clang/opt +usr.bin/clang/llvm-mc +usr.bin/clang/llvm-modextract +usr.bin/clang/llvm-nm +usr.bin/clang/llvm-objdump +usr.bin/clang/llvm-pdbutil +usr.bin/clang/llvm-profdata +usr.bin/clang/llvm-rtdyld +usr.bin/clang/llvm-symbolizer +usr.bin/clang/llvm-tblgen +usr.bin/clang/llvm-xray +usr.bin/cmp +usr.bin/col +usr.bin/colldef +usr.bin/colrm +usr.bin/column +usr.bin/comm +usr.bin/compile_et +usr.bin/compress +usr.bin/cpio +usr.bin/cpio/tests +usr.bin/cpuset +usr.bin/csplit +usr.bin/ctags +usr.bin/ctlstat +usr.bin/cut +usr.bin/cxxfilt +usr.bin/dc +usr.bin/diff +usr.bin/diff3 +usr.bin/dirname +usr.bin/dpv +usr.bin/drill +usr.bin/dtc +usr.bin/du +usr.bin/ee +usr.bin/elf2aout +usr.bin/elfcopy +usr.bin/elfdump +usr.bin/enigma +usr.bin/env +usr.bin/expand +usr.bin/factor +usr.bin/false +usr.bin/fetch +usr.bin/file +usr.bin/file2c +usr.bin/find +usr.bin/finger +usr.bin/fmt +usr.bin/fold +usr.bin/fortune/fortune +usr.bin/fortune/strfile +usr.bin/fortune/unstr +usr.bin/from +usr.bin/fstat +usr.bin/fsync +usr.bin/ftp +usr.bin/gcore +usr.bin/gencat +usr.bin/getaddrinfo +usr.bin/getconf +usr.bin/getconf/tests +usr.bin/getent +usr.bin/getopt +usr.bin/gprof +usr.bin/grdc +usr.bin/grep +usr.bin/gzip +usr.bin/head +usr.bin/hesinfo +usr.bin/hexdump +usr.bin/host +usr.bin/iconv +usr.bin/id +usr.bin/ident +usr.bin/indent +usr.bin/ipcrm +usr.bin/ipcs +usr.bin/iscsictl +usr.bin/join +usr.bin/jot +usr.bin/kdump +usr.bin/keylogin +usr.bin/keylogout +usr.bin/killall +usr.bin/ktrace +usr.bin/ktrdump +usr.bin/lam +usr.bin/last +usr.bin/lastcomm +usr.bin/ldd +usr.bin/leave +usr.bin/less +usr.bin/lessecho +usr.bin/lesskey +usr.bin/lex +usr.bin/limits +usr.bin/locale +usr.bin/locate/bigram +usr.bin/locate/code +usr.bin/locate/locate +usr.bin/localedef +usr.bin/lock +usr.bin/lockf +usr.bin/logger +usr.bin/login +usr.bin/logins +usr.bin/logname +usr.bin/look +usr.bin/lsvfs +usr.bin/lzmainfo +usr.bin/m4 +usr.bin/mail +usr.bin/mandoc +usr.bin/mesg +usr.bin/minigzip +usr.bin/ministat +usr.bin/mkcsmapper +usr.bin/mkcsmapper_static +usr.bin/mkesdb +usr.bin/mkesdb_static +usr.bin/mkfifo +usr.bin/mkimg +usr.bin/mklocale +usr.bin/mkstr +usr.bin/mktemp +usr.bin/mkuzip +usr.bin/mt +usr.bin/morse +usr.bin/msgs +usr.bin/nc +usr.bin/ncal +usr.bin/netstat +usr.bin/newgrp +usr.bin/newkey +usr.bin/nfsstat +usr.bin/nice +usr.bin/nl +usr.bin/nm +usr.bin/nohup +usr.bin/numactl +usr.bin/number +usr.bin/opieinfo +usr.bin/opiekey +usr.bin/opiepasswd +usr.bin/pamtest +usr.bin/passwd +usr.bin/paste +usr.bin/patch +usr.bin/pathchk +usr.bin/perror +usr.bin/pom +usr.bin/pr +usr.bin/primes +usr.bin/printenv +usr.bin/printf +usr.bin/proccontrol +usr.bin/procstat +usr.bin/procstat/tests +usr.bin/rev +usr.bin/protect +usr.bin/quota +usr.bin/random +usr.bin/rctl +usr.bin/readelf +usr.bin/renice +usr.bin/resizewin +usr.bin/revoke +usr.bin/rsh +usr.bin/rpcgen +usr.bin/rpcinfo +usr.bin/rs +usr.bin/xlint/lint1 +usr.bin/xlint/lint2 +usr.bin/xlint/xlint +usr.bin/rup +usr.bin/ruptime +usr.bin/rusers +usr.bin/rwall +usr.bin/rwho +usr.bin/script +usr.bin/sdiff +usr.bin/sed +usr.bin/seq +usr.bin/setchannel +usr.bin/showmount +usr.bin/size +usr.bin/smbutil +usr.bin/sockstat +usr.bin/soelim +usr.bin/sort +usr.bin/split +usr.bin/stat +usr.bin/stdbuf +usr.bin/strings +usr.bin/su +usr.bin/svn/svn +usr.bin/svn/svnadmin +usr.bin/svn/svnbench +usr.bin/svn/svndumpfilter +usr.bin/svn/svnfsfs +usr.bin/svn/svnlook +usr.bin/svn/svnmucc +usr.bin/svn/svnrdump +usr.bin/svn/svnserve +usr.bin/svn/svnsync +usr.bin/svn/svnversion +usr.bin/systat +usr.bin/tabs +usr.bin/tail +usr.bin/talk +usr.bin/tar +usr.bin/tar/tests +usr.bin/tcopy +usr.bin/tee +usr.bin/telnet +usr.bin/tftp +usr.bin/time +usr.bin/timeout +usr.bin/tip/tip +usr.bin/top +usr.bin/touch +usr.bin/tput +usr.bin/tr +usr.bin/true +usr.bin/truncate +usr.bin/truss +usr.bin/tset +usr.bin/tsort +usr.bin/tty +usr.bin/ul +usr.bin/uname +usr.bin/unexpand +usr.bin/unifdef +usr.bin/uniq +usr.bin/units +usr.bin/unvis +usr.bin/unzip +usr.bin/usbhidaction +usr.bin/usbhidctl +usr.bin/users +usr.bin/uudecode +usr.bin/uuencode +usr.bin/vacation +usr.bin/vgrind +usr.bin/vi +usr.bin/vis +usr.bin/vmstat +usr.bin/vtfontcvt +usr.bin/w +usr.bin/wall +usr.bin/wc +usr.bin/what +usr.bin/whereis +usr.bin/which +usr.bin/who +usr.bin/whois +usr.bin/write +usr.bin/xargs +usr.bin/xinstall +usr.bin/catman +usr.bin/xo +usr.bin/xstr +usr.bin/xz +usr.bin/xzdec +usr.bin/yes +usr.bin/yacc +usr.bin/ypcat +usr.bin/ypmatch +usr.bin/ypwhich +usr.bin/zstd +usr.bin/xopo +usr.bin/sdiotool +usr.sbin/ac +usr.sbin/accton +usr.sbin/acpi/acpiconf +usr.sbin/acpi/acpidb +usr.sbin/acpi/acpidump +usr.sbin/acpi/iasl +usr.sbin/amd/amd +usr.sbin/amd/amq +usr.sbin/amd/fixmount +usr.sbin/amd/fsinfo +usr.sbin/amd/hlfsd +usr.sbin/amd/mk-amd-map +usr.sbin/amd/pawd +usr.sbin/amd/wire-test +usr.sbin/ancontrol +usr.sbin/apm +usr.sbin/apmd +usr.sbin/arp +usr.sbin/asf +usr.sbin/audit +usr.sbin/auditd +usr.sbin/auditdistd +usr.sbin/auditreduce +usr.sbin/authpf +usr.sbin/autofs +usr.sbin/bhyve +usr.sbin/bhyvectl +usr.sbin/bhyveload +usr.sbin/binmiscctl +usr.sbin/blacklistctl +usr.sbin/blacklistd +usr.sbin/bluetooth/ath3kfw +usr.sbin/bluetooth/bcmfw +usr.sbin/bluetooth/bt3cfw +usr.sbin/bluetooth/bthidcontrol +usr.sbin/bluetooth/bthidd +usr.sbin/bluetooth/btpand +usr.sbin/bluetooth/hccontrol +usr.sbin/bluetooth/hcsecd +usr.sbin/bluetooth/hcseriald +usr.sbin/bluetooth/l2control +usr.sbin/bluetooth/l2ping +usr.sbin/bluetooth/rfcomm_pppd +usr.sbin/bluetooth/sdpcontrol +usr.sbin/bluetooth/sdpd +usr.sbin/boot0cfg +usr.sbin/bootparamd/bootparamd +usr.sbin/bootparamd/callbootd +usr.sbin/bsdinstall/distextract +usr.sbin/bsdinstall/distfetch +usr.sbin/bsdinstall/partedit +usr.sbin/bsnmpd/bsnmpd +usr.sbin/bsnmpd/gensnmptree +usr.sbin/bsnmpd/tools/bsnmptools +usr.sbin/btxld +usr.sbin/camdd +usr.sbin/cdcontrol +usr.sbin/chkgrp +usr.sbin/chown +usr.sbin/chroot +usr.sbin/ckdist +usr.sbin/clear_locks +usr.sbin/config +usr.sbin/cpucontrol +usr.sbin/cron/cron +usr.sbin/cron/crontab +usr.sbin/crunch/crunchgen +usr.sbin/crunch/crunchide +usr.sbin/crunch/examples +usr.sbin/ctladm +usr.sbin/ctld +usr.sbin/ctm/ctm +usr.sbin/ctm/ctm_dequeue +usr.sbin/ctm/ctm_rmail +usr.sbin/ctm/ctm_smail +usr.sbin/ctm/mkCTM +usr.sbin/cxgbetool +usr.sbin/daemon +usr.sbin/dconschat +usr.sbin/devctl +usr.sbin/devinfo +usr.sbin/digictl +usr.sbin/diskinfo +usr.sbin/dumpcis +usr.sbin/editmap +usr.sbin/edquota +usr.sbin/eeprom +usr.sbin/efidp +usr.sbin/efivar +usr.sbin/extattr +usr.sbin/extattrctl +usr.sbin/fdcontrol +usr.sbin/fdformat +usr.sbin/fdread +usr.sbin/fdwrite +usr.sbin/fifolog/fifolog_create +usr.sbin/fifolog/fifolog_reader +usr.sbin/fifolog/fifolog_writer +usr.sbin/flowctl +usr.sbin/fmtree +usr.sbin/fstyp +usr.sbin/ftp-proxy +usr.sbin/fwcontrol +usr.sbin/getfmac +usr.sbin/getpmac +usr.sbin/gpioctl +usr.sbin/gssd +usr.sbin/gstat +usr.sbin/hyperv/tools/kvp +usr.sbin/hyperv/tools/vss +usr.sbin/i2c +usr.sbin/ifmcstat +usr.sbin/inetd +usr.sbin/iostat +usr.sbin/iovctl +usr.sbin/ip6addrctl +usr.sbin/ipfwpcap +usr.sbin/iscsid +usr.sbin/jail +usr.sbin/jexec +usr.sbin/jls +usr.sbin/kbdcontrol +usr.sbin/kbdmap +usr.sbin/keyserv +usr.sbin/kgmon +usr.sbin/kgzip +usr.sbin/kldxref +usr.sbin/lastlogin +usr.sbin/lmcconfig +usr.sbin/lpr/chkprintcap +usr.sbin/lpr/filters.ru/koi2855 +usr.sbin/lpr/filters.ru/koi2alt +usr.sbin/lpr/filters +usr.sbin/lpr/lpc +usr.sbin/lpr/lpd +usr.sbin/lpr/lpq +usr.sbin/lpr/lpr +usr.sbin/lpr/lprm +usr.sbin/lpr/lptest +usr.sbin/lpr/pac +usr.sbin/lptcontrol +usr.sbin/mailstats +usr.sbin/mailwrapper +usr.sbin/makefs +usr.sbin/makemap +usr.sbin/memcontrol +usr.sbin/mfiutil +usr.sbin/mixer +usr.sbin/mld6query +usr.sbin/mlxcontrol +usr.sbin/mount_smbfs +usr.sbin/mountd +usr.sbin/moused +usr.sbin/mpsutil +usr.sbin/mptable +usr.sbin/mptutil +usr.sbin/mtest +usr.sbin/nandsim +usr.sbin/nandtool +usr.sbin/ndiscvt +usr.sbin/ndp +usr.sbin/newsyslog +usr.sbin/nfscbd +usr.sbin/nfsd +usr.sbin/nfsdumpstate +usr.sbin/nfsrevoke +usr.sbin/nfsuserd +usr.sbin/ngctl +usr.sbin/nghook +usr.sbin/nmtree +usr.sbin/nologin +usr.sbin/nscd +usr.sbin/ntp/ntp-keygen +usr.sbin/ntp/ntpd +usr.sbin/ntp/ntpdate +usr.sbin/ntp/ntpdc +usr.sbin/ntp/ntpq +usr.sbin/ntp/ntptime +usr.sbin/ntp/sntp +usr.sbin/nvram +usr.sbin/ofwdump +usr.sbin/pciconf +usr.sbin/pkg +usr.sbin/pmcannotate +usr.sbin/pmccontrol +usr.sbin/pmcstat +usr.sbin/pmcstudy +usr.sbin/pnpinfo +usr.sbin/portsnap/make_index +usr.sbin/portsnap/phttpget +usr.sbin/powerd +usr.sbin/ppp +usr.sbin/pppctl +usr.sbin/praliases +usr.sbin/praudit +usr.sbin/procctl +usr.sbin/prometheus_sysctl_exporter +usr.sbin/pstat +usr.sbin/pw +usr.sbin/pw/tests +usr.sbin/pwd_mkdb +usr.sbin/quot +usr.sbin/quotaon +usr.sbin/rarpd +usr.sbin/repquota +usr.sbin/rip6query +usr.sbin/rmt +usr.sbin/route6d +usr.sbin/rpc.lockd +usr.sbin/sa +usr.sbin/rpc.statd +usr.sbin/rpc.umntall +usr.sbin/rpc.yppasswdd +usr.sbin/rpc.ypupdated +usr.sbin/rpc.ypxfrd +usr.sbin/rpcbind +usr.sbin/rrenumd +usr.sbin/rtadvctl +usr.sbin/rtadvd +usr.sbin/rtprio +usr.sbin/rtsold +usr.sbin/rwhod +usr.sbin/sendmail +usr.sbin/services_mkdb +usr.sbin/sesutil +usr.sbin/setfib +usr.sbin/setfmac +usr.sbin/setpmac +usr.sbin/smbmsg +usr.sbin/snapinfo +usr.sbin/spray +usr.sbin/syslogd +usr.sbin/tcpdchk +usr.sbin/tcpdmatch +usr.sbin/tcpdrop +usr.sbin/tcpdump/tcpdump +usr.sbin/timed/timed +usr.sbin/timed/timedc +usr.sbin/traceroute +usr.sbin/traceroute6 +usr.sbin/trpt +usr.sbin/tzsetup +usr.sbin/uathload +usr.sbin/uefisign +usr.sbin/ugidfw +usr.sbin/uhsoctl +usr.sbin/unbound/anchor +usr.sbin/unbound/checkconf +usr.sbin/unbound/control +usr.sbin/unbound/daemon +usr.sbin/usbconfig +usr.sbin/usbdump +usr.sbin/utx +usr.sbin/vidcontrol +usr.sbin/vipw +usr.sbin/wake +usr.sbin/watch +usr.sbin/watchdogd +usr.sbin/wlandebug +usr.sbin/wpa/hostapd +usr.sbin/wpa/hostapd_cli +usr.sbin/wpa/ndis_events +usr.sbin/wpa/wpa_cli +usr.sbin/wpa/wpa_passphrase +usr.sbin/wpa/wpa_priv +usr.sbin/wpa/wpa_supplicant +usr.sbin/yp_mkdb +usr.sbin/ypbind +usr.sbin/ypldap +usr.sbin/yppoll +usr.sbin/yppush +usr.sbin/ypserv +usr.sbin/ypset +usr.sbin/zic/zdump +usr.sbin/zic/zic +usr.sbin/zonectl Index: tools/tools/smoketestsuite/utils.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/utils.h @@ -0,0 +1,83 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _UTILS_H_ +#define _UTILS_H_ + +#include +#include + +namespace utils { + /* + * Option relation which maps option names to + * a unique identifier in their description. + */ + struct OptRelation { + char type; /* Option type: (s)short/(l)long. */ + std::string value; /* Name of the option. */ + /* The keyword which should be looked up in usage + * message (if) produced when using this options. + */ + std::string keyword; + }; + + /* + * Read/Write file descriptors for a pipe. + */ + struct PipeDescriptor { + int readfd; + int writefd; + pid_t pid; /* PID of the forked shell process. */ + }; + + /* + * Temporary directory inside which the utility-specific + * commands will be executed, and all the side effects + * (core dumps, executables etc.) that are created will + * be restricted in this directory. + */ + extern const char *tmpdir; + + std::string GenerateCommand(std::string, std::string); + std::pair Execute(std::string); + PipeDescriptor* POpen(const char*); + + class OptDefinition { + public: + /* List of all the accepted options with unknown usage. */ + std::vector opt_list; + /* Map "option value" to "option definition". */ + std::unordered_map opt_map; + std::unordered_map::iterator opt_map_iter; + + void InsertOpts(); + std::vector CheckOpts(std::string); + }; +} + +#endif /* _UTILS_H_ */ Index: tools/tools/smoketestsuite/utils.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/utils.cpp @@ -0,0 +1,336 @@ +/*- + * Copyright 2017-2018 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "utils.h" +#include "fetch_groff.h" +#include "logging.h" + +#define READ 0 /* Pipe descriptor: read end. */ +#define WRITE 1 /* Pipe descriptor: write end. */ +/* + * Buffer size (used for buffering output generated + * after executing the utility-specific command). + */ +#define BUFSIZE 128 +#define TIMEOUT 1 /* Threshold (seconds) for a function call to return. */ + +const char *utils::tmpdir = "tmpdir"; +/* + * Insert a list of user-defined option definitions + * into a hashmap. These specific option definitions + * are the ones which one can be easily tested. + */ +void +utils::OptDefinition::InsertOpts() +{ + /* Option definitions. */ + OptRelation h_def; /* '-h' */ + h_def.type = 's'; + h_def.value = "h"; + h_def.keyword = "help"; + + OptRelation v_def; /* '-v' */ + v_def.type = 's'; + v_def.value = "v"; + v_def.keyword = "version"; + + /* + * "opt_map" contains all the options + * which can be easily tested. + */ + opt_map.insert(std::make_pair + ("h", (OptRelation)h_def)); + opt_map.insert(std::make_pair + ("v", (OptRelation)v_def)); +}; + +/* + * For the utility under test, find the supported options + * present in the hashmap generated by InsertOpts() + * and return them in a form of list of option relations. + */ +std::vector +utils::OptDefinition::CheckOpts(std::string utility) +{ + std::string opt_id = ".It Fl"; /* Option identifier in man page. */ + std::string line; /* An individual line in a man-page. */ + std::string opt_name; /* Name of the option. */ + std::string buffer; /* Option description extracted from man-page. */ + std::string opt_string; /* Identified option names. */ + int opt_pos; /* Starting index of the (identified) option. */ + int space_index; /* First occurrence of space in option definition. */ + std::vector identified_opts; + std::vector supported_sections = { "1", "8" }; + + /* Generate the hashmap "opt_map". */ + InsertOpts(); + std::ifstream infile(groff::groff_map[utility]); + + /* + * Search for all the options accepted by the + * utility and collect those present in "opt_map". + */ + while (std::getline(infile, line)) { + if ((opt_pos = line.find(opt_id)) != std::string::npos) { + /* Locate the position of option name. */ + opt_pos += opt_id.size() + 1; + if (opt_pos > line.size()) { + /* + * This condition will trigger when a utility + * supports an empty argument, e.g. tset(issue #9) + */ + continue; + } + + /* + * Check for long options ; While here, also sanitize + * multi-word option definitions in a man page to properly + * extract short options from option definitions such as: + * .It Fl r Ar seconds (taken from date(1)). + */ + if ((space_index = line.find(" ", opt_pos + 1, 1)) + != std::string::npos) + opt_name = line.substr(opt_pos, space_index - opt_pos); + else + opt_name = line.substr(opt_pos); + + /* + * Check if the identified option matches the identifier. + * "opt_list.back()" is the previously checked option, the + * description of which is now stored in "buffer". + */ + if (!opt_list.empty() && + (opt_map_iter = opt_map.find(opt_list.back())) + != opt_map.end() && + buffer.find((opt_map_iter->second).keyword) != std::string::npos) { + identified_opts.push_back(&(opt_map_iter->second)); + /* + * Since the usage of the option under test + * is known, we remove it from "opt_list". + */ + opt_list.pop_back(); + } + + /* Update the list of valid options. */ + opt_list.push_back(opt_name); + /* Empty the buffer for next option's description. */ + buffer.clear(); + } else { + /* + * Collect the option description until next + * valid option definition is encountered. + */ + buffer.append(line); + } + } + + return identified_opts; +} + +/* Generates command for execution. */ +std::string +utils::GenerateCommand(std::string utility, std::string opt) +{ + std::string command = utility; + + if (!opt.empty()) + command += " -" + opt; + command += " 2>&1 readfd = pdes[READ]; + pipe_descr->writefd = pdes[WRITE]; + + /* Type-cast to avoid compiler warnings [-Wwrite-strings]. */ + argv[0] = (char *)"sh"; + argv[1] = (char *)"-c"; + argv[2] = (char *)command; + argv[3] = NULL; + + switch (child_pid = vfork()) { + case -1: /* Error. */ + free(pipe_descr); + close(pdes[READ]); + close(pdes[WRITE]); + return NULL; + case 0: /* Child. */ + /* + * TODO Verify if the following operations on both read + * and write file-descriptors (irrespective of the value + * of 'type', which is no longer being used) is safe. + */ + if (pdes[WRITE] != STDOUT_FILENO) + dup2(pdes[WRITE], STDOUT_FILENO); + else + fcntl(pdes[WRITE], F_SETFD, 0); + if (pdes[READ] != STDIN_FILENO) + dup2(pdes[READ], STDIN_FILENO); + else + fcntl(pdes[READ], F_SETFD, 0); + + /* + * For current usecase, it might so happen that the child gets + * stuck on a blocking read (e.g. passwd(1)) waiting for user + * input. In that case the child will be killed via a signal. + * To avoid any effect on the parent's execution, we place the + * child in a separate process group with pgid set as "child_pid". + */ + setpgid(child_pid, child_pid); + execve("/bin/sh", argv, NULL); + exit(127); + } + + pipe_descr->pid = child_pid; + return pipe_descr; +} + +/* + * Executes the command passed as argument in a + * shell and returns its output and exit status. + */ +std::pair +utils::Execute(std::string command) +{ + int result; + int exitstatus; + std::array buffer; + std::string usage_output; + struct timeval tv; + fd_set readfds; + FILE *pipe; + PipeDescriptor *pipe_descr; + pid_t pid; + int pstat; + + /* Execute "command" inside "tmpdir". */ + chdir(tmpdir); + pipe_descr = utils::POpen(command.c_str()); + chdir(".."); + + if (pipe_descr == NULL) { + logging::LogPerror("utils::POpen()"); + exit(EXIT_FAILURE); + } + + /* Close the unrequired file-descriptor. */ + close(pipe_descr->writefd); + + pipe = fdopen(pipe_descr->readfd, "r"); + free(pipe_descr); + if (pipe == NULL) { + close(pipe_descr->readfd); + logging::LogPerror("fdopen()"); + exit(EXIT_FAILURE); + } + + /* Set a timeout for shell process to complete its execution. */ + tv.tv_sec = TIMEOUT; + tv.tv_usec = 0; + + FD_ZERO(&readfds); + FD_SET(fileno(pipe), &readfds); + result = select(fileno(pipe) + 1, &readfds, NULL, NULL, &tv); + + if (result > 0) { + while (!feof(pipe) && !ferror(pipe)) { + if (fgets(buffer.data(), BUFSIZE, pipe) != NULL) + usage_output += buffer.data(); + } + } else if (result == -1) { + logging::LogPerror("select()"); + if (kill(pipe_descr->pid, SIGTERM) < 0) + logging::LogPerror("kill()"); + } else if (result == 0) { + /* + * We gave a relaxed value of TIMEOUT seconds for the shell process + * to complete it's execution. If at this point it is still + * alive, it (most probably) is stuck on a blocking read + * waiting for the user input. Since a few of the utilities + * performing such blocking reads don't respond to SIGINT + * (e.g. pax(1)), we terminate the shell process via SIGTERM. + */ + if (kill(pipe_descr->pid, SIGTERM) < 0) + logging::LogPerror("kill()"); + } + + /* Retrieve exit status of the shell process. */ + do { + pid = wait4(pipe_descr->pid, &pstat, 0, (struct rusage *)0); + } while (pid == -1 && errno == EINTR); + + fclose(pipe); + exitstatus = (pid == -1) ? -1 : WEXITSTATUS(pstat); + DEBUGP("Command: %s, exit status: %d\n", command.c_str(), exitstatus); + + return std::make_pair + ((std::string)usage_output, (int)exitstatus); +}