Index: tools/tools/smoketestsuite/Makefile =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/Makefile @@ -0,0 +1,47 @@ +# $FreeBSD$ +# +# Makefile for building the test generation tool + +CC= c++ +CFLAGS=-I/usr/local/include -std=c++11 +LIBS= -L/usr/local/lib \ + -lboost_filesystem \ + -lboost_system +OBJS= logging.o \ + utils.o \ + read_annotations.o \ + generate_license.o \ + add_testcase.o \ + fetch_groff.o \ + generate_test.o + +generate_tests: $(OBJS) + $(CC) $(CFLAGS) -o $(.TARGET) $(.ALLSRC) $(LIBS) + +$(OBJS): $(.PREFIX).cpp $(.PREFIX).h + $(CC) $(CFLAGS) -c $(.PREFIX).cpp + +.PHONY: clean \ + fetch_groff \ + fetch_utils \ + run + +clean: + rm -rf $(OBJS) \ + generate_tests \ + scripts/utils_list \ + groff \ + failed_groff \ + generated_tests + +fetch_groff: + sh scripts/fetch_groff.sh + +fetch_utils: + sh scripts/fetch_utils.sh + +run: + @echo Generating annotations... + sh scripts/generate_annot.sh + @echo Generating test files... + ./generate_tests Index: tools/tools/smoketestsuite/README =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/README @@ -0,0 +1,46 @@ +$FreeBSD$ + +Smoke testing of base utilities +=============================== + +Test generation tool made as a part of Google Summer of Code '17 with FreeBSD (https://summerofcode.withgoogle.com/projects/#6426676740227072). +Refer the FreeBSD wiki (https://wiki.freebsd.org/SummerOfCode2017/SmokeTestingOfBaseUtilities) for an overview and updates. +A brief description and implementation details: https://lists.freebsd.org/pipermail/soc-status/2017-July/001079.html. +The diagram "architecture.png" briefly summarizes how different components fit with the testcase-generator. + +Directory Structure +=================== + +. +├── annotations +│   └── ........................:: Annotation files (generated/user-defined) +├── scripts +│   └── ........................:: Helper scripts +├── architecture.png ...........:: A brief architecture diagram +├── add_testcase.cpp ...........:: Testcase generator +├── generate_license.cpp .......:: Customized license generator +├── generate_test.cpp ..........:: Test generator +├── read_annotations.cpp .......:: Annotation parser +└── utils.cpp ..................:: Index generator + +- - - + +Dependencies +============ +* Boost C++ libraries: The tool was tested to work with the port "boost-all-1.64.0". + +Instructions +============ +* The directory "groff" should be populated with the relevant groff scripts before proceeding for test generation. These scripts are available in the FreeBSD source tree. + +* For populating the directory "groff", execute the following from the project root - + + make fetch_utils + make fetch_groff + +Generating tests +================ +Execute the following from the project root - + + make clean + make && make run Index: tools/tools/smoketestsuite/add_testcase.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/add_testcase.h @@ -0,0 +1,38 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +namespace addtestcase { + void KnownTestcase(std::string, std::string, std::string, \ + std::string, std::ofstream&); + + void UnknownTestcase(std::string, std::string, std::string, \ + int, std::string&); + + void NoArgsTestcase(std::string, std::pair, \ + std::ofstream&); +} Index: tools/tools/smoketestsuite/add_testcase.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/add_testcase.cpp @@ -0,0 +1,171 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include "add_testcase.h" + +/* Adds a test-case for an option with known usage. */ +void +addtestcase::KnownTestcase(std::string option, + std::string util_with_section, + std::string descr, + std::string output, + std::ofstream& test_script) +{ + std::string testcase_name; + std::string utility = util_with_section.substr(0, + util_with_section.length() - 3); + + /* Add testcase name. */ + test_script << "atf_test_case "; + if (!option.empty()) { + testcase_name = option; + testcase_name.append("_flag"); + } else + testcase_name = "no_arguments"; + test_script << testcase_name + "\n"; + + /* Add testcase description. */ + test_script << testcase_name + + "_head()\n{\n\tatf_set \"descr\" "; + if (!descr.empty()) + test_script << descr; + else + test_script << "\"Verify the usage of option \'" + option + "\'\""; + test_script << "\n}\n\n"; + + /* Add body of the testcase. */ + test_script << testcase_name + "_body()\n{" + + "\n\tatf_check -s exit:0 -o "; + + /* Match the usage output if generated. */ + if (!output.empty()) + test_script << "inline:\"" + output + "\" "; + else + test_script << "empty "; + test_script << utility; + + if (!option.empty()) + test_script << " -" + option; + test_script << "\n}\n\n"; +} + +/* Adds a test-case for an option with unknown usage. */ +void +addtestcase::UnknownTestcase(std::string option, + std::string util_with_section, + std::string output, + int exitstatus, + std::string& testcase_buffer) +{ + std::string utility = util_with_section.substr(0, + util_with_section.length() - 3); + + testcase_buffer.append("\n\tatf_check -s not-exit:0 -e "); + + /* Check if a usage message was produced (case-insensitive match). */ + if (boost::iequals(output.substr(0, 6), "usage:")) + testcase_buffer.append("match:\"$usage_output\" "); + else if (!output.empty()) + testcase_buffer.append("inline:\"" + output + "\" "); + else + testcase_buffer.append("empty "); + + testcase_buffer.append(utility); + + if (!option.empty()) + testcase_buffer.append(" -" + option); +} + +/* Adds a test-case for usage without any arguments. */ +void +addtestcase::NoArgsTestcase(std::string util_with_section, + std::pair output, + std::ofstream& test_script) +{ + std::string descr; + std::string utility = util_with_section.substr(0, + util_with_section.length() - 3); + + if (output.second) { + /* An error was encountered. */ + test_script << std::string("atf_test_case no_arguments\n") + + "no_arguments_head()\n{\n\tatf_set \"descr\" "; + if (!output.first.empty()) { + /* + * We expect a usage message to be generated in this case + * (case-insensitive match). + */ + if (boost::iequals(output.first.substr(0, 6), "usage:")) { + descr = "\"Verify that " + util_with_section + + " fails and generates a valid usage \" \\\n\t\t\t" + + "\"message when no arguments are supplied\""; + + test_script << descr + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e match:\"$usage_output\" " + + utility; + } else { + descr = "\"Verify that " + util_with_section + + " fails and generates a valid output \" \\\n\t\t\t" + + "\"when no arguments are supplied\""; + + test_script << descr + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e inline:\"" + + output.first + "\" " + + utility; + } + } else { + descr = "\"Verify that " + util_with_section + + " fails silently when no arguments are supplied\"" ; + test_script << descr + "\n}\n\nno_arguments_body()\n{" + + "\n\tatf_check -s not-exit:0 -e empty " + + utility; + } + + test_script << "\n}\n\n"; + } else { + /* + * The command ran successfully, hence we guessed + * a correct usage for the utility under test. + */ + if (!output.first.empty()) + descr = "\"Verify that " + util_with_section + + " executes successfully and produces a valid \" \\\n\t\t\t" + + "\"output when invoked without any arguments\""; + else + descr = "\"Verify that " + util_with_section + + " executes successfully and silently \" \\\n\t\t\t" + + "\"when invoked without any arguments\""; + + addtestcase::KnownTestcase("", util_with_section, descr, + output.first, test_script); + } +} Index: tools/tools/smoketestsuite/annotations/date_test.annot =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/annotations/date_test.annot @@ -0,0 +1,5 @@ +R_flag +j_flag +n_flag +no_arguments +u_flag Index: tools/tools/smoketestsuite/fetch_groff.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/fetch_groff.h @@ -0,0 +1,33 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ +#include + +namespace groff { + extern std::unordered_map util_path_map; + int FetchGroffScripts(); +} Index: tools/tools/smoketestsuite/fetch_groff.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/fetch_groff.cpp @@ -0,0 +1,113 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "fetch_groff.h" + +/* Hashmap mapping utility name to its location in src tree. */ +std::unordered_map groff::util_path_map; + +/* + * Traverses the FreeBSD src tree looking for groff + * scripts for section 1 and section 8 utilities and + * copies them to the directory "groff/". + * TODO Use the paths of scripts instead of copying. + */ +int +groff::FetchGroffScripts() +{ + std::string utils_list = "scripts/utils_list"; + std::string groff_src = "groff/"; + std::string freebsd_src = "../../../"; + std::string util_dir; + std::string pathname; + std::ifstream utils_fstream; + std::regex section_1 ("(.*).1"); + std::regex section_8 ("(.*).8"); + struct stat sb; + struct dirent *ent; + DIR *dir; + + /* Check if the file "scripts/utils_list" exists. */ + if (stat(utils_list.c_str(), &sb) != 0) { + std::cerr << "scripts/utils_list does not exists.\n" + "Run 'make fetch_utils' first.\n"; + return -1; + } + utils_fstream.open(utils_list); + + /* Remove the state "groff" directory and create an empty one. */ + boost::filesystem::remove_all(groff_src); + boost::filesystem::create_directory(groff_src); + + while(getline(utils_fstream, util_dir)) { + pathname = freebsd_src + util_dir + "/tests"; + + /* + * Copy the groff script only if the utility does not + * already have tests, i.e. the "tests" directory is absent. + */ + if (!(stat(pathname.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode))) { + pathname = freebsd_src + util_dir; + util_path_map[util_dir.substr(util_dir.find_last_of("/") + 1)] = pathname; + + if ((dir = opendir(pathname.c_str())) != NULL) { + ent = readdir(dir); /* Skip directory entry for "." */ + ent = readdir(dir); /* Skip directory entry for ".." */ + + while ((ent = readdir(dir)) != NULL) { + /* Copy the groff scripts for section 1 and section 8 utilities. */ + if (std::regex_match(ent->d_name, section_1) || + std::regex_match(ent->d_name, section_8)) { + std::ifstream src(pathname + "/" + ent->d_name, std::ios::binary); + std::ofstream dst(groff_src + ent->d_name, std::ios::binary); + dst << src.rdbuf(); + } + } + + closedir(dir); + } else { + perror("opendir()"); + } + } + } + + /* Remove non-executable utilities. */ + boost::filesystem::remove(groff_src + "elfcopy.1"); + std::cout << "Successfully updated 'groff/'\n"; + utils_fstream.close(); + + return 0; +} Index: tools/tools/smoketestsuite/generate_license.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_license.h @@ -0,0 +1,31 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +namespace generatelicense { + std::string GenerateLicense(int, char **); +} Index: tools/tools/smoketestsuite/generate_license.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_license.cpp @@ -0,0 +1,82 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include "generate_license.h" +#include "utils.h" + +std::string +generatelicense::GenerateLicense(int argc, char **argv) +{ + std::string license; + std::string copyright_owner; + + if (argc > 1) { + if (argc == 3 && strncmp(argv[1], "--name ", 7)) + copyright_owner = argv[2]; + else { + std::cerr << "Usage: ./generate_tests --name \n"; + exit(EXIT_FAILURE); + } + } else + copyright_owner = utils::Execute("id -P | cut -d : -f 8").first; + + license = + "#\n" + "# Copyright 2017 " + copyright_owner + "\n" + "# All rights reserved.\n" + "#\n" + "# Redistribution and use in source and binary forms, with or without\n" + "# modification, are permitted provided that the following conditions\n" + "# are met:\n" + "# 1. Redistributions of source code must retain the above copyright\n" + "# notice, this list of conditions and the following disclaimer.\n" + "# 2. Redistributions in binary form must reproduce the above copyright\n" + "# notice, this list of conditions and the following disclaimer in the\n" + "# documentation and/or other materials provided with the distribution.\n" + "#\n" + "# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND\n" + "# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n" + "# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n" + "# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n" + "# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n" + "# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n" + "# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n" + "# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n" + "# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n" + "# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n" + "# SUCH DAMAGE.\n" + "#\n" + "# $FreeBSD$\n" + "#\n\n"; + + return license; +} Index: tools/tools/smoketestsuite/generate_test.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_test.h @@ -0,0 +1,36 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "utils.h" + +namespace generatetest { + void IntHandler(int); + void GenerateMakefile(std::string, std::string); + void GenerateTest(std::string, std::string, + std::string&, const char*); +} Index: tools/tools/smoketestsuite/generate_test.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/generate_test.cpp @@ -0,0 +1,390 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "add_testcase.h" +#include "fetch_groff.h" +#include "generate_license.h" +#include "generate_test.h" +#include "read_annotations.h" + +void +generatetest::IntHandler(int dummmy) +{ + std::cerr << "\nExiting...\n"; + /* Remove the temporary directory. */ + boost::filesystem::remove_all(utils::tmpdir); + exit(EXIT_FAILURE); +} + +/* [Batch mode] Generate a makefile for the test of given utility. */ +void +generatetest::GenerateMakefile(std::string utility, std::string util_dir) +{ + std::ofstream makefile_fstream; + + makefile_fstream.open(util_dir + "/Makefile", std::ios::out); + makefile_fstream << "# $FreeBSD$\n\nATF_TESTS_SH+= " + + utility + "_test\n\n" + + ".include \n"; + makefile_fstream.close(); +} + +/* Generate a test for the given utility. */ +void +generatetest::GenerateTest(std::string utility, + std::string section, + std::string& license, + const char *tests_dir) +{ + std::list identified_opt_list; /* List of identified option relations. */ + /* + * Vector of usage messages -- used for validating + * their consistency across different runs. + */ + std::vector usage_messages; + std::string command; /* (Utility-specific) command to be executed. */ + std::string testcase_list; /* List of testcases. */ + std::string testcase_buffer; /* Buffer for (temporarily) holding testcase data. */ + std::string test_file; /* atf-sh test name. */ + std::string util_with_section; /* Section number appended to utility. */ + std::ofstream test_fstream; /* Output stream for the atf-sh test. */ + std::pair output; /* Return value type for `Execute()`. */ + std::unordered_set annotation_set; /* Hashset of utility specific annotations. */ + /* Number of options for which a testcase has been generated. */ + int progress = 0; + + /* Read annotations and populate hash set "annotation_set". */ + annotations::read_annotations(utility, annotation_set); + util_with_section = utility + '(' + section + ')'; + utils::OptDefinition opt_def; + identified_opt_list = opt_def.CheckOpts(utility); + test_file = tests_dir + utility + "_test.sh"; + + /* Indicate the start of test generation for current utility. */ + if (isatty(fileno(stderr))) { + std::cerr << std::setw(18) << util_with_section << " | " + << progress << "/" << opt_def.opt_list.size() << "\r"; + } + + /* Add license in the generated test scripts. */ + test_fstream.open(test_file, std::ios::out); + test_fstream << license; + + /* + * If a known option was encountered (i.e. `identified_opt_list` is + * populated), produce a testcase to check the validity of the + * result of that option. If no known option was encountered, + * produce testcases to verify the correct (generated) usage + * message when using the supported options incorrectly. + */ + + /* Add testcases for known options. */ + if (!identified_opt_list.empty()) { + for (const auto &i : identified_opt_list) { + command = utility + " -" + i->value + " 2>&1 value, util_with_section, output.first, + output.second, testcase_buffer); + } else { + addtestcase::KnownTestcase(i->value, util_with_section, + "", output.first, test_fstream); + } + testcase_list.append("\tatf_add_test_case " + i->value + "_flag\n"); + } + } + + /* Add testcases for the options whose usage is not yet known. */ + if (!opt_def.opt_list.empty()) { + /* + * For the purpose of adding a "$usage_output" variable, + * we choose the option which produces one. + * TODO(shivansh) Avoid double executions of an option, i.e. one while + * selecting usage message and another while generating testcase. + */ + if (opt_def.opt_list.size() == 1) { + /* Utility supports a single option, check if it produces a usage message. */ + command = utility + " -" + opt_def.opt_list.front() + " 2>&1 &1 &1 &1 > util_vector; + std::vector>::iterator util_vector_iter; + std::string test_file; /* atf-sh test name. */ + std::string util_name; /* Utility name. */ + struct stat sb; + struct dirent *ent; + DIR *groff_dir_ptr; + char answer; /* User input to determine actions to be taken. */ + std::string license; /* Customized license generated during runtime. */ + std::string util_dir; /* Path to utility in src tree. */ + /* Directory for collecting groff scripts for utilities with failed test generation. */ + const char *failed_groff_dir = "failed_groff/"; + const char *groff_dir = "groff/"; /* Directory of groff scripts. */ + const char *tests_dir = "generated_tests/"; /* Directory to collect generated tests. */ + /* + * Instead of generating tests for all the utilities, "batch mode" + * allows generation of tests for first "batch_limit" number of + * utilities selected from "scripts/utils_list". + */ + bool batch_mode = false; + int batch_limit; /* Number of tests to be generated in batch mode. */ + + /* Handle interrupts. */ + signal(SIGINT, generatetest::IntHandler); + + /* + * Create a temporary directory where all the + * side-effects introduced by utility-specific + * commands are restricted. + */ + boost::filesystem::create_directory(utils::tmpdir); + + std::cout << "\nInstead of generating tests for all the utilities, 'batch mode'\n" + "allows generation of tests for first few utilities selected from\n" + "'scripts/utils_list', and places them at their correct location\n" + "in the src tree, with corresponding makefiles created.\n" + "Run in 'batch mode' ? [y/N] "; + std::cin.get(answer); + + switch(answer) { + case 'y': + case 'Y': + batch_mode = true; + if (groff::FetchGroffScripts() == -1) + return EXIT_FAILURE; + break; + case '\n': + default: + break; + } + if (batch_mode) { + std::cout << "Number of utilities to select for test generation: "; + std::cin >> batch_limit; + + if (batch_limit <= 0) { + std::cerr << "Invalid number given as input. Exiting...\n"; + return EXIT_FAILURE; + } + } else { + /* Check if the directory "groff/" is populated with groff scripts. */ + std::cout << "Update groff directory ? [y/N] "; + std::cin.get(answer); + + switch(answer) { + case 'y': + case 'Y': + if (groff::FetchGroffScripts() == -1) + return EXIT_FAILURE; + break; + case '\n': + default: + break; + } + } + + /* + * For testing (or generating tests for only selected utilities), + * "util_vector" can be populated above during declaration. + */ + if (util_vector.empty()) { + if ((groff_dir_ptr = opendir(groff_dir))) { + readdir(groff_dir_ptr); /* Skip directory entry for "." */ + readdir(groff_dir_ptr); /* Skip directory entry for ".." */ + while ((ent = readdir(groff_dir_ptr))) { + util_name = ent->d_name; + util_vector.push_back(std::make_pair + (util_name.substr(0, util_name.length() - 2), + util_name.substr(util_name.length() - 1, 1))); + } + closedir(groff_dir_ptr); + } else { + fprintf(stderr, "Could not open the directory: groff/"); + return EXIT_FAILURE; + } + } + + /* Check if the directory "tests_dir" exists. */ + if (stat(tests_dir, &sb) || !S_ISDIR(sb.st_mode)) { + boost::filesystem::path dir(tests_dir); + if (boost::filesystem::create_directory(dir)) + std::cout << "Directory created: " << tests_dir << std::endl; + else { + std::cerr << "Unable to create directory: " << tests_dir << std::endl; + return EXIT_FAILURE; + } + } + + /* Generate a license to be added in the generated scripts. */ + license = generatelicense::GenerateLicense(argc, argv); + + boost::filesystem::remove_all(failed_groff_dir); + boost::filesystem::create_directory(failed_groff_dir); + + /* Generate a tabular-like format. */ + std::cout << std::endl; + std::cout << std::setw(21) << "Utility | " << "Progress\n"; + std::cout << std::setw(32) << "----------+-----------\n"; + + if (batch_mode) { + /* + * Generate tests for first "batch_limit" number of + * utilities selected from "scripts/utils_list". + */ + for (util_vector_iter = util_vector.begin(); + util_vector_iter != util_vector.begin() + batch_limit; + util_vector_iter++) { + util_dir = groff::util_path_map.at(util_vector_iter->first) + "/tests/"; + boost::filesystem::remove_all(util_dir); + boost::filesystem::create_directory(util_dir); + generatetest::GenerateMakefile(util_vector_iter->first, util_dir); + + test_file = tests_dir + util_vector_iter->first + "_test.sh"; + generatetest::GenerateTest(util_vector_iter->first, + util_vector_iter->second, + license, util_dir.c_str()); + } + } else { + for (const auto &util : util_vector) { + /* TODO(shivansh) Check before overwriting existing test scripts. */ + test_file = tests_dir + util.first + "_test.sh"; + generatetest::GenerateTest(util.first, util.second, + license, tests_dir); + } + } + + /* Remove the temporary directory. */ + boost::filesystem::remove_all(utils::tmpdir); + + return EXIT_SUCCESS; +} Index: tools/tools/smoketestsuite/logging.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/logging.h @@ -0,0 +1,40 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* Use a gcc variadic macro to conditionally compile debug printing. */ +#ifdef DEBUG +#define DEBUGP(...) \ + fprintf(stdout, __VA_ARGS__); \ + fflush(stdout); +#else +#define DEBUGP(...) {} +#endif + +namespace logging { + void LogPerror(std::string); +} Index: tools/tools/smoketestsuite/logging.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/logging.cpp @@ -0,0 +1,39 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include + +#include "logging.h" + +void +logging::LogPerror(std::string message) +{ +#ifdef DEBUG + perror(message.c_str()); +#endif +} Index: tools/tools/smoketestsuite/read_annotations.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/read_annotations.h @@ -0,0 +1,34 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include + +namespace annotations { + void read_annotations(std::string, \ + std::unordered_set&); +} Index: tools/tools/smoketestsuite/read_annotations.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/read_annotations.cpp @@ -0,0 +1,62 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +#include "read_annotations.h" + +/* + * Read the annotation files and skip + * generation of the respective tests. + */ +void +annotations::read_annotations(std::string utility, + std::unordered_set& annotation_set) +{ + std::string line; + std::ifstream annotation_fstream; + annotation_fstream.open("annotation_set/" + utility + "_test.annot"); + + while (getline(annotation_fstream, line)) { + /* Add a unique identifier for no_arguments testcase */ + if (!line.compare(0, 12, "no_arguments")) + annotation_set.insert("*"); + + /* + * Add flag value for supported argument testcases + * Doing so we ignore the "invalid_usage" testcase + * as it is guaranteed to always succeed. + */ + else if (!line.compare(2, 4, "flag")) + annotation_set.insert(line.substr(0, 1)); + } + + annotation_fstream.close(); +} Index: tools/tools/smoketestsuite/scripts/README =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/README @@ -0,0 +1,8 @@ +# Helper scripts + +Script Name | Functionality +------------------+----------------- +fetch_groff.sh | Lists location of all section 1 utilities +fetch_utils.sh | Saves all the base utilities in the src tree in **utils_list** +generate_annot.sh | Populates annotation files under [annotations](../annotations) +validate.sh | Validates side-effects of newly introduced changes in the tool Index: tools/tools/smoketestsuite/scripts/fetch_utils.sh =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/fetch_utils.sh @@ -0,0 +1,43 @@ +#! /bin/sh +# +# Copyright 2017 Shivansh Rai +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +# Script for listing all the base utilities + +set -e +script_dir="scripts" +src="$(dirname $0)/../../../../" + +fetch_utils() { + cd "$src" + find . -name Makefile | xargs grep -l 'PROG\|PROG_CXX' \ + | sed -e 's|/Makefile$||' | cut -c 3- +} + +rm -f "$script_dir/utils_list" + +(fetch_utils) >> "$script_dir/utils_list" Index: tools/tools/smoketestsuite/scripts/generate_annot.sh =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/scripts/generate_annot.sh @@ -0,0 +1,91 @@ +#!/bin/sh +# +# Copyright 2017 Shivansh Rai +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +# Script for generating annotations based on generated tests. + +suffix="_test" +extension=".sh" +update_required=0 + +message="Following annotation files were updated. \n" + +printf " ++--------------------------------------------------+ +| Checking if any annotation file is to be updated | ++--------------------------------------------------+\n" + +for f in "generated_tests"/* +do + annotations="" + file=$(basename "$f") + test=${file%$extension} + utility=${test%$suffix} + test_dir="/usr/tests/bin/$utility" + + ( + cd "$test_dir" 2>/dev/null || exit + report=$(kyua report) + i=2 + + while true + do + testcase=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $1}') + status=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $3}') + check=$(printf "%s" "$testcase" | cut -s -f1 -d":") + + if [ "$check" != "$test" ]; then + if [ "$annotations" ]; then + if [ $update_required = 0 ]; then + printf $message + update_required=1 + fi + + annotations_file="annotations/$test.annot" + # Append only the new annotations + printf "$annotations" > "$annotations_file.temp" + [ ! -e "$annotations_file" ] && touch "$annotations_file" + comm -13 "$annotations_file" "$annotations_file.temp" >> \ + "$annotations_file" && printf "\t%s\n" "annotations/$test.annot" + rm -f "$annotations_file.temp" + fi + + break + fi + + if [ "$status" = "failed:" ]; then + testcase=${testcase#"$test:"} + annotations="$annotations$testcase\n" + fi + + i=$((i+1)) + done + ) + +done + +printf "==============================================================================\n\n" Index: tools/tools/smoketestsuite/utils.h =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/utils.h @@ -0,0 +1,79 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include + +namespace utils { + /* + * Option relation which maps option names to + * a unique identifier in their description. + */ + struct OptRelation { + char type; /* Option type: (s)short/(l)long. */ + std::string value; /* Name of the option. */ + /* The keyword which should be looked up in usage + * message (if) produced when using this options. + */ + std::string keyword; + }; + + /* + * Read/Write file descriptors for a pipe. + */ + struct PipeDescriptor { + int readfd; + int writefd; + pid_t pid; /* PID of the forked shell process. */ + }; + + /* + * Temporary directory inside which the utility-specific + * commands will be executed, and all the side effects + * (core dumps, executables etc.) that are created will + * be restricted in this directory. + */ + extern const char *tmpdir; + + std::pair Execute(std::string); + PipeDescriptor* POpen(const char*); + + class OptDefinition { + public: + /* List of all the accepted options with unknown usage. */ + std::list opt_list; + /* Map "option value" to "option definition". */ + std::unordered_map opt_map; + std::unordered_map::iterator opt_map_iter; + + void InsertOpts(); + std::list CheckOpts(std::string); + }; +} Index: tools/tools/smoketestsuite/utils.cpp =================================================================== --- /dev/null +++ tools/tools/smoketestsuite/utils.cpp @@ -0,0 +1,323 @@ +/*- + * Copyright 2017 Shivansh Rai + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils.h" +#include "logging.h" + +#define READ 0 /* Pipe descriptor: read end. */ +#define WRITE 1 /* Pipe descriptor: write end. */ +/* + * Buffer size (used for buffering output generated + * after executing the utility-specific command). + */ +#define BUFSIZE 128 +#define TIMEOUT 1 /* Threshold (seconds) for a function call to return. */ + +const char *utils::tmpdir = "tmpdir"; +/* + * Insert a list of user-defined option definitions + * into a hashmap. These specific option definitions + * are the ones which one can be easily tested. + */ +void +utils::OptDefinition::InsertOpts() +{ + /* Option definitions. */ + OptRelation h_def; /* '-h' */ + h_def.type = 's'; + h_def.value = "h"; + h_def.keyword = "help"; + + OptRelation v_def; /* '-v' */ + v_def.type = 's'; + v_def.value = "v"; + v_def.keyword = "version"; + + /* + * "opt_map" contains all the options + * which can be easily tested. + */ + opt_map.insert(std::make_pair + ("h", (OptRelation)h_def)); + opt_map.insert(std::make_pair + ("v", (OptRelation)v_def)); +}; + +/* + * For the utility under test, find the supported options + * present in the hashmap generated by InsertOpts() + * and return them in a form of list of option relations. + */ +std::list +utils::OptDefinition::CheckOpts(std::string utility) +{ + std::string line; /* An individual line in a man-page. */ + std::string opt_name; /* Name of the option. */ + std::string opt_identifier = ".It Fl"; /* Identifier for an option in man page. */ + std::string buffer; /* Option description extracted from man-page. */ + std::string opt_string; /* Identified option names. */ + int opt_position; /* Starting index of the (identified) option. */ + int space_index; /* First occurrence of space character + * in a multi-word option definition. + */ + std::list identified_opt_list; /* List of identified option relations. */ + std::list supported_sections = { "1", "8" }; + + /* Generate the hashmap opt_map. */ + InsertOpts(); + + for (const auto §ion : supported_sections) { + std::ifstream infile("groff/" + utility + "." + section); + + /* + * Search for all the options accepted by the + * utility and collect those present in "opt_map". + */ + while (std::getline(infile, line)) { + if ((opt_position = line.find(opt_identifier)) != std::string::npos) { + /* Locate the position of option name. */ + opt_position += opt_identifier.length() + 1; + + if (opt_position > line.length()) { + /* + * This condition will trigger when a utility + * supports an empty argument, e.g. tset(issue #9) + */ + continue; + } + + /* + * Check for long options ; While here, also sanitize + * multi-word option definitions in a man page to properly + * extract short options from option definitions such as: + * .It Fl r Ar seconds (taken from date(1)). + */ + if ((space_index = line.find(" ", opt_position + 1, 1)) + != std::string::npos) + opt_name = line.substr(opt_position, space_index - opt_position); + else + opt_name = line.substr(opt_position); + + /* + * Check if the identified option matches the identifier. + * "opt_list.back()" is the previously checked option, the + * description of which is now stored in "buffer". + */ + if (!opt_list.empty() && + (opt_map_iter = opt_map.find(opt_list.back())) + != opt_map.end() && + buffer.find((opt_map_iter->second).keyword) != std::string::npos) { + identified_opt_list.push_back(&(opt_map_iter->second)); + + /* + * Since the usage of the option under test + * is known, we remove it from "opt_list". + */ + opt_list.pop_back(); + } + + /* Update the list of valid options. */ + opt_list.push_back(opt_name); + + /* Empty the buffer for next option's description. */ + buffer.clear(); + } else { + /* + * Collect the option description until next + * valid option definition is encountered. + */ + buffer.append(line); + } + } + } + + return identified_opt_list; +} + +/* + * When pclose() is called on the stream returned by popen(), + * it waits indefinitely for the created shell process to + * terminate in cases where the shell command waits for the + * user input via a blocking read (e.g. passwd(1)). + * Hence, we define a custom function which alongside + * returning the read-write file descriptors, also returns + * the pid of the newly created (child) shell process. This + * pid can later be used for sending a signal to the child. + * + * NOTE For the sake of completeness, we also specify actions + * to be taken corresponding to the "w" (write) type. However, + * in this context only the "r" (read) type will be required. + */ + +utils::PipeDescriptor* +utils::POpen(const char *command) +{ + int pdes[2]; + char *argv[4]; + pid_t child_pid; + PipeDescriptor *pipe_descr = (PipeDescriptor *)malloc(sizeof(PipeDescriptor)); + + /* Create a pipe with ~ + * - pdes[READ]: read end + * - pdes[WRITE]: write end + */ + if (pipe2(pdes, O_CLOEXEC) < 0) + return NULL; + + pipe_descr->readfd = pdes[READ]; + pipe_descr->writefd = pdes[WRITE]; + + argv[0] = (char *)"sh"; /* Type-cast to avoid compiler warning [-Wwrite-strings]. */ + argv[1] = (char *)"-c"; + argv[2] = (char *)command; + argv[3] = NULL; + + switch (child_pid = vfork()) { + case -1: /* Error. */ + free(pipe_descr); + close(pdes[READ]); + close(pdes[WRITE]); + return NULL; + case 0: /* Child. */ + /* + * TODO Verify if the following operations on both read + * and write file-descriptors (irrespective of the value + * of 'type', which is no longer being used) is safe. + */ + if (pdes[WRITE] != STDOUT_FILENO) + dup2(pdes[WRITE], STDOUT_FILENO); + else + fcntl(pdes[WRITE], F_SETFD, 0); + if (pdes[READ] != STDIN_FILENO) + dup2(pdes[READ], STDIN_FILENO); + else + fcntl(pdes[READ], F_SETFD, 0); + + /* + * For current usecase, it might so happen that the child gets + * stuck on a blocking read (e.g. passwd(1)) waiting for user + * input. In that case the child will be killed via a signal. + * To avoid any effect on the parent's execution, we place the + * child in a separate process group with pgid set as "child_pid". + */ + setpgid(child_pid, child_pid); + execve("/bin/sh", argv, NULL); + exit(127); + } + + pipe_descr->pid = child_pid; + + return pipe_descr; +} + +/* + * Executes the passed argument "command" in a shell + * and returns its output and the exit status. + */ +std::pair +utils::Execute(std::string command) +{ + int result; + std::array buffer; + std::string usage_output; + struct timeval tv; + fd_set readfds; + FILE *pipe; + PipeDescriptor *pipe_descr; + + DEBUGP("Executing: %s\n", command.c_str()); + + /* Execute "command" inside "tmpdir". */ + chdir(tmpdir); + pipe_descr = utils::POpen(command.c_str()); + chdir(".."); + + if (pipe_descr == NULL) { + logging::LogPerror("utils::POpen()"); + exit(EXIT_FAILURE); + } + + /* Close the unrequired file-descriptor. */ + close(pipe_descr->writefd); + + pipe = fdopen(pipe_descr->readfd, "r"); + free(pipe_descr); + if (pipe == NULL) { + close(pipe_descr->readfd); + logging::LogPerror("fdopen()"); + exit(EXIT_FAILURE); + } + + /* + * Set a timeout value for the spawned shell + * process to complete its execution. + */ + tv.tv_sec = TIMEOUT; + tv.tv_usec = 0; + + FD_ZERO(&readfds); + FD_SET(fileno(pipe), &readfds); + result = select(fileno(pipe) + 1, &readfds, NULL, NULL, &tv); + + if (result > 0) { + while (!feof(pipe) && !ferror(pipe)) { + if (fgets(buffer.data(), BUFSIZE, pipe) != NULL) + usage_output += buffer.data(); + } + } else if (result == -1) { + logging::LogPerror("select()"); + if (kill(pipe_descr->pid, SIGTERM) < 0) + logging::LogPerror("kill()"); + } else if (result == 0) { + /* + * We gave a relaxed value of TIMEOUT seconds for the shell process + * to complete it's execution. If at this point it is still + * alive, it (most probably) is stuck on a blocking read + * waiting for the user input. Since a few of the utilities + * performing such blocking reads don't respond to SIGINT + * (e.g. pax(1)), we terminate the shell process via SIGTERM. + */ + if (kill(pipe_descr->pid, SIGTERM) < 0) + logging::LogPerror("kill()"); + } + + return std::make_pair + ((std::string)usage_output, WEXITSTATUS(pclose(pipe))); +}