Page MenuHomeFreeBSD
Files F137777291

D12249.id35835.diff
No OneTemporary
Actions

D12249.id35835.diff
View Options

Index: tools/tools/smoketestsuite/Makefile
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/Makefile
@@ -0,0 +1,45 @@
+# $FreeBSD$
+#
+# Makefile for building the test generation tool
+
+CC= c++
+CFLAGS=-I/usr/local/include -std=c++11
+LIBS= -L/usr/local/lib \
+ -lboost_system \
+ -lboost_filesystem
+OBJS= utils.o \
+ read_annotations.o \
+ generate_license.o \
+ add_testcase.o \
+ generate_test.o
+
+generate_tests: $(OBJS)
+ $(CC) $(CFLAGS) -o $(.TARGET) $(.ALLSRC) $(LIBS)
+
+$(OBJS): $(.PREFIX).cpp $(.PREFIX).h
+ $(CC) $(CFLAGS) -c $(.PREFIX).cpp
+
+.PHONY: clean \
+ fetch_groff \
+ fetch_utils \
+ run
+
+clean:
+ rm -rf $(OBJS) \
+ generate_tests \
+ scripts/utils_list \
+ groff \
+ failed_groff \
+ generated_tests
+
+fetch_groff:
+ sh scripts/fetch_groff.sh
+
+fetch_utils:
+ sh scripts/fetch_utils.sh
+
+run:
+ @echo Generating annotations...
+ sh scripts/generate_annot.sh
+ @echo Generating test files...
+ ./generate_tests
Index: tools/tools/smoketestsuite/README
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/README
@@ -0,0 +1,46 @@
+$FreeBSD$
+
+Smoke testing of base utilities
+===============================
+
+Test generation tool made as a part of Google Summer of Code '17 with FreeBSD (https://summerofcode.withgoogle.com/projects/#6426676740227072).
+Refer the FreeBSD wiki (https://wiki.freebsd.org/SummerOfCode2017/SmokeTestingOfBaseUtilities) for an overview and updates.
+A brief description and implementation details: https://lists.freebsd.org/pipermail/soc-status/2017-July/001079.html.
+The diagram "architecture.png" briefly summarizes how different components fit with the testcase-generator.
+
+Directory Structure
+===================
+
+.
+├── annotations
+│   └── ........................:: Annotation files (generated/user-defined)
+├── scripts
+│   └── ........................:: Helper scripts
+├── architecture.png ...........:: A brief architecture diagram
+├── add_testcase.cpp ...........:: Testcase generator
+├── generate_license.cpp .......:: Customized license generator
+├── generate_test.cpp ..........:: Test generator
+├── read_annotations.cpp .......:: Annotation parser
+└── utils.cpp ..................:: Index generator
+
+- - -
+
+Dependencies
+============
+* Boost C++ libraries: The tool was tested to work with the port "boost-all-1.64.0".
+
+Instructions
+============
+* The directory "groff" should be populated with the relevant groff scripts before proceeding for test generation. These scripts are available in the FreeBSD source tree.
+
+* For populating the directory "groff", execute the following from the project root -
+
+ make fetch_utils
+ make fetch_groff
+
+Generating tests
+================
+Execute the following from the project root -
+
+ make clean
+ make && make run
Index: tools/tools/smoketestsuite/add_testcase.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/add_testcase.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+namespace addtestcase {
+ void KnownTestcase(std::string, std::string, std::string, \
+ std::string, std::ofstream&);
+
+ void UnknownTestcase(std::string, std::string, std::string, \
+ int, std::string&);
+
+ void NoArgsTestcase(std::string, std::pair<std::string, int>, \
+ std::ofstream&);
+}
Index: tools/tools/smoketestsuite/add_testcase.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/add_testcase.cpp
@@ -0,0 +1,171 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <boost/algorithm/string.hpp>
+#include <fstream>
+#include <iostream>
+
+#include "add_testcase.h"
+
+/* Adds a test-case for an option with known usage. */
+void
+addtestcase::KnownTestcase(std::string option,
+ std::string util_with_section,
+ std::string descr,
+ std::string output,
+ std::ofstream& test_script)
+{
+ std::string testcase_name;
+ std::string utility = util_with_section.substr(0,
+ util_with_section.length() - 3);
+
+ /* Add testcase name. */
+ test_script << "atf_test_case ";
+ if (!option.empty()) {
+ testcase_name = option;
+ testcase_name.append("_flag");
+ } else
+ testcase_name = "no_arguments";
+ test_script << testcase_name + "\n";
+
+ /* Add testcase description. */
+ test_script << testcase_name
+ + "_head()\n{\n\tatf_set \"descr\" ";
+ if (!descr.empty())
+ test_script << descr;
+ else
+ test_script << "\"Verify the usage of option \'" + option + "\'\"";
+ test_script << "\n}\n\n";
+
+ /* Add body of the testcase. */
+ test_script << testcase_name + "_body()\n{"
+ + "\n\tatf_check -s exit:0 -o ";
+
+ /* Match the usage output if generated. */
+ if (!output.empty())
+ test_script << "inline:\"" + output + "\" ";
+ else
+ test_script << "empty ";
+ test_script << utility;
+
+ if (!option.empty())
+ test_script << " -" + option;
+ test_script << "\n}\n\n";
+}
+
+/* Adds a test-case for an option with unknown usage. */
+void
+addtestcase::UnknownTestcase(std::string option,
+ std::string util_with_section,
+ std::string output,
+ int exitstatus,
+ std::string& testcase_buffer)
+{
+ std::string utility = util_with_section.substr(0,
+ util_with_section.length() - 3);
+
+ testcase_buffer.append("\n\tatf_check -s not-exit:0 -e ");
+
+ /* Check if a usage message was produced (case-insensitive match). */
+ if (boost::iequals(output.substr(0, 6), "usage:"))
+ testcase_buffer.append("match:\"$usage_output\" ");
+ else if (!output.empty())
+ testcase_buffer.append("inline:\"" + output + "\" ");
+ else
+ testcase_buffer.append("empty ");
+
+ testcase_buffer.append(utility);
+
+ if (!option.empty())
+ testcase_buffer.append(" -" + option);
+}
+
+/* Adds a test-case for usage without any arguments. */
+void
+addtestcase::NoArgsTestcase(std::string util_with_section,
+ std::pair<std::string, int> output,
+ std::ofstream& test_script)
+{
+ std::string descr;
+ std::string utility = util_with_section.substr(0,
+ util_with_section.length() - 3);
+
+ if (output.second) {
+ /* An error was encountered. */
+ test_script << std::string("atf_test_case no_arguments\n")
+ + "no_arguments_head()\n{\n\tatf_set \"descr\" ";
+ if (!output.first.empty()) {
+ /*
+ * We expect a usage message to be generated in this case
+ * (case-insensitive match).
+ */
+ if (boost::iequals(output.first.substr(0, 6), "usage:")) {
+ descr = "\"Verify that " + util_with_section
+ + " fails and generates a valid usage"
+ + " message when no arguments are supplied\"";
+
+ test_script << descr + "\n}\n\nno_arguments_body()\n{"
+ + "\n\tatf_check -s not-exit:0 -e match:\"$usage_output\" "
+ + utility;
+ } else {
+ descr = "\"Verify that " + util_with_section
+ + " fails and generates a valid output"
+ + " when no arguments are supplied\"";
+
+ test_script << descr + "\n}\n\nno_arguments_body()\n{"
+ + "\n\tatf_check -s not-exit:0 -e inline:\""
+ + output.first + "\" "
+ + utility;
+ }
+ } else {
+ descr = "\"Verify that " + util_with_section
+ + " fails silently when no arguments are supplied\"" ;
+ test_script << descr + "\n}\n\nno_arguments_body()\n{"
+ + "\n\tatf_check -s not-exit:0 -e empty "
+ + utility;
+ }
+
+ test_script << "\n}\n\n";
+ } else {
+ /*
+ * The command ran successfully, hence we guessed
+ * a correct usage for the utility under test.
+ */
+ if (!output.first.empty())
+ descr = "\"Verify that " + util_with_section
+ + " executes successfully and produces a valid"
+ + " output when invoked without any arguments\"";
+ else
+ descr = "\"Verify that " + util_with_section
+ + " executes successfully and silently"
+ + " when invoked without any arguments\"";
+
+ addtestcase::KnownTestcase("", util_with_section, descr,
+ output.first, test_script);
+ }
+}
Index: tools/tools/smoketestsuite/annotations/date_test.annot
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/annotations/date_test.annot
@@ -0,0 +1,5 @@
+R_flag
+j_flag
+n_flag
+no_arguments
+u_flag
Index: tools/tools/smoketestsuite/generate_license.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_license.h
@@ -0,0 +1,31 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+namespace generatelicense {
+ std::string GenerateLicense(int, char **);
+}
Index: tools/tools/smoketestsuite/generate_license.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_license.cpp
@@ -0,0 +1,82 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+
+#include "generate_license.h"
+#include "utils.h"
+
+std::string
+generatelicense::GenerateLicense(int argc, char **argv)
+{
+ std::string license;
+ std::string copyright_owner;
+
+ if (argc > 1) {
+ if (argc == 3 && strncmp(argv[1], "--name ", 7))
+ copyright_owner = argv[2];
+ else {
+ std::cerr << "Usage: ./generate_tests --name <copyright_owner>\n";
+ exit(EXIT_FAILURE);
+ }
+ } else
+ copyright_owner = utils::Execute("id -P | cut -d : -f 8").first;
+
+ license =
+ "#\n"
+ "# Copyright 2017 " + copyright_owner + "\n"
+ "# All rights reserved.\n"
+ "#\n"
+ "# Redistribution and use in source and binary forms, with or without\n"
+ "# modification, are permitted provided that the following conditions\n"
+ "# are met:\n"
+ "# 1. Redistributions of source code must retain the above copyright\n"
+ "# notice, this list of conditions and the following disclaimer.\n"
+ "# 2. Redistributions in binary form must reproduce the above copyright\n"
+ "# notice, this list of conditions and the following disclaimer in the\n"
+ "# documentation and/or other materials provided with the distribution.\n"
+ "#\n"
+ "# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND\n"
+ "# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n"
+ "# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n"
+ "# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n"
+ "# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n"
+ "# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n"
+ "# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n"
+ "# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n"
+ "# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n"
+ "# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"
+ "# SUCH DAMAGE.\n"
+ "#\n"
+ "# $FreeBSD$\n"
+ "#\n\n";
+
+ return license;
+}
Index: tools/tools/smoketestsuite/generate_test.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_test.h
@@ -0,0 +1,33 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "utils.h"
+
+namespace generatetest {
+ void GenerateTest(std::string, std::string, std::string&);
+}
Index: tools/tools/smoketestsuite/generate_test.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_test.cpp
@@ -0,0 +1,272 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <boost/algorithm/string.hpp>
+#include <boost/filesystem.hpp>
+#include <cstdio>
+#include <cstdlib>
+#include <dirent.h>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <pthread.h>
+#include <signal.h>
+#include <stdexcept>
+#include <sys/stat.h>
+#include <unordered_set>
+
+#include "add_testcase.h"
+#include "generate_license.h"
+#include "generate_test.h"
+#include "read_annotations.h"
+
+const char *tests_dir = "generated_tests/"; /* Directory to collect generated tests. */
+
+/* Generate a test for the given utility. */
+void
+generatetest::GenerateTest(std::string utility,
+ std::string section,
+ std::string& license)
+{
+ std::list<utils::OptRelation *> identified_opt_list; /* List of identified option relations. */
+ std::vector<std::string> usage_messages; /* Vector of usage messages. Used for validating
+ * their consistency across different runs.
+ */
+ std::string command; /* (Utility-specific) command to be executed. */
+ std::string testcase_list; /* List of testcases. */
+ std::string testcase_buffer; /* Buffer for (temporarily) holding testcase data. */
+ std::string test_file; /* atf-sh test name. */
+ std::string util_with_section; /* Section number appended to utility. */
+ std::ofstream test_fstream; /* Output stream for the atf-sh test. */
+ std::pair<std::string, int> output; /* Return value type for `Execute()`. */
+ std::unordered_set<std::string> annotation_set; /* Hashset of utility specific annotations. */
+
+ /* Read annotations and populate hash set "annotation_set". */
+ annotations::read_annotations(utility, annotation_set);
+
+ util_with_section = utility + '(' + section + ')';
+
+ utils::OptDefinition opt_def;
+ identified_opt_list = opt_def.CheckOpts(utility);
+ test_file = tests_dir + utility + "_test.sh";
+
+ /* Add license in the generated test scripts. */
+ test_fstream.open(test_file, std::ios::out);
+ test_fstream << license;
+
+ /*
+ * If a known option was encountered (i.e. `identified_opt_list` is
+ * populated), produce a testcase to check the validity of the
+ * result of that option. If no known option was encountered,
+ * produce testcases to verify the correct (generated) usage
+ * message when using the supported options incorrectly.
+ */
+
+ /* Add testcases for known options. */
+ if (!identified_opt_list.empty()) {
+ for (const auto &i : identified_opt_list) {
+ command = utility + " -" + i->value + " 2>&1 </dev/null";
+ output = utils::Execute(command);
+ if (boost::iequals(output.first.substr(0, 6), "usage:")) {
+ /* A usage message was produced => our guessed usage is incorrect. */
+ addtestcase::UnknownTestcase(i->value, util_with_section, output.first,
+ output.second, testcase_buffer);
+ } else {
+ addtestcase::KnownTestcase(i->value, util_with_section,
+ NULL, output.first, test_fstream);
+ }
+ testcase_list.append("\tatf_add_test_case " + i->value + "_flag\n");
+ }
+ }
+
+ /* Add testcases for the options whose usage is not yet known. */
+ if (!opt_def.opt_list.empty()) {
+ /*
+ * For the purpose of adding a "$usage_output" variable,
+ * we choose the option which produces one.
+ * TODO(shivansh) Avoid double executions of an option, i.e. one while
+ * selecting usage message and another while generating testcase.
+ */
+
+ if (opt_def.opt_list.size() == 1) {
+ /* Utility supports a single option, check if it produces a usage message. */
+ command = utility + " -" + opt_def.opt_list.front() + " 2>&1 </dev/null";
+ output = utils::Execute(command);
+
+ if (output.second && !output.first.empty())
+ test_fstream << "usage_output=\'" + output.first + "\'\n\n";
+ } else {
+ /*
+ * Utility supports multiple options. In case the usage message
+ * is consistent for atleast "two" options, we reduce duplication
+ * by assigning a variable "usage_output" in the test script.
+ */
+ for (const auto &i : opt_def.opt_list) {
+ command = utility + " -" + i + " 2>&1 </dev/null";
+ output = utils::Execute(command);
+
+ if (output.second && usage_messages.size() < 3)
+ usage_messages.push_back(output.first);
+ }
+
+ for (int j = 0; j < usage_messages.size(); j++) {
+ if (!(usage_messages.at(j)).compare(usage_messages.at((j+1) % usage_messages.size())) &&
+ !output.first.empty()) {
+ test_fstream << "usage_output=\'"
+ + output.first.substr(0, 7 + utility.size())
+ + "\'\n\n";
+ break;
+ }
+ }
+ }
+
+ /*
+ * Execute the utility with supported options
+ * and add (+ve)/(-ve) testcases accordingly.
+ */
+ for (const auto &i : opt_def.opt_list) {
+ /* If the option is annotated, ignore it. */
+ if (annotation_set.find(i) != annotation_set.end())
+ continue;
+
+ command = utility + " -" + i + " 2>&1 </dev/null";
+ output = utils::Execute(command);
+
+ if (output.second) {
+ /* Non-zero exit status was encountered. */
+ addtestcase::UnknownTestcase(i, util_with_section, output.first,
+ output.second, testcase_buffer);
+ } else {
+ /*
+ * EXIT_SUCCESS was encountered. Hence,
+ * the guessed usage was correct.
+ */
+ addtestcase::KnownTestcase(i, util_with_section, "",
+ output.first, test_fstream);
+ testcase_list.append(std::string("\tatf_add_test_case ")
+ + i + "_flag\n");
+ }
+ }
+
+ testcase_list.append("\tatf_add_test_case invalid_usage\n");
+ test_fstream << std::string("atf_test_case invalid_usage\ninvalid_usage_head()\n")
+ + "{\n\tatf_set \"descr\" \"Verify that an invalid usage "
+ + "with a supported option produces a valid error message"
+ + "\"\n}\n\ninvalid_usage_body()\n{";
+
+ test_fstream << testcase_buffer + "\n}\n\n";
+ }
+
+ /*
+ * Add a testcase under "no_arguments" for
+ * running the utility without any arguments.
+ */
+ if (annotation_set.find("*") == annotation_set.end()) {
+ command = utility + " 2>&1 </dev/null";
+ output = utils::Execute(command);
+ addtestcase::NoArgsTestcase(util_with_section, output, test_fstream);
+ testcase_list.append("\tatf_add_test_case no_arguments\n");
+ }
+
+ test_fstream << "atf_init_test_cases()\n{\n" + testcase_list + "}\n";
+ test_fstream.close();
+}
+
+int
+main(int argc, char **argv)
+{
+ std::ifstream groff_list;
+ std::list<std::pair<std::string, std::string>> utility_list;
+ std::string test_file; /* atf-sh test name. */
+ std::string util_name; /* Utility name. */
+ struct stat sb;
+ struct dirent *ent;
+ DIR *groff_dir_ptr;
+ char answer; /* User input to determine overwriting of test files. */
+ std::string license; /* Customized license generated during runtime. */
+ const char *failed_groff_dir = "failed_groff/"; /* Directory for collecting groff scripts
+ * for utilities with failed test generation.
+ */
+ const char *groff_dir = "groff/"; /* Directory of groff scripts. */
+
+ /*
+ * For testing (or generating tests for only selected utilities),
+ * the utility_list can be populated above during declaration.
+ */
+ if (utility_list.empty()) {
+ if ((groff_dir_ptr = opendir(groff_dir))) {
+ readdir(groff_dir_ptr); /* Skip directory entry for "." */
+ readdir(groff_dir_ptr); /* Skip directory entry for ".." */
+ while ((ent = readdir(groff_dir_ptr))) {
+ util_name = ent->d_name;
+ utility_list.push_back(std::make_pair<std::string, std::string>
+ (util_name.substr(0, util_name.length() - 2),
+ util_name.substr(util_name.length() - 1, 1)));
+ }
+ closedir(groff_dir_ptr);
+ } else {
+ fprintf(stderr, "Could not open the directory: ./groff\nRefer to the "
+ "section \"Populating groff scripts\" in README!\n");
+ return EXIT_FAILURE;
+ }
+ }
+
+ /* Check if the directory "tests_dir" exists. */
+ if (stat(tests_dir, &sb) || !S_ISDIR(sb.st_mode)) {
+ boost::filesystem::path dir(tests_dir);
+ if (boost::filesystem::create_directory(dir))
+ std::cout << "Directory created: " << tests_dir << std::endl;
+ else {
+ std::cerr << "Unable to create directory: " << tests_dir << std::endl;
+ return EXIT_FAILURE;
+ }
+ }
+
+ /* Generate a license to be added in the generated scripts. */
+ license = generatelicense::GenerateLicense(argc, argv);
+
+ remove(failed_groff_dir);
+ boost::filesystem::create_directory(failed_groff_dir);
+
+ /* Generate a tabular-like format. */
+ std::cout << std::endl;
+ std::cout << std::setw(21) << "Utility | " << "Progress\n";
+ std::cout << std::setw(32) << "----------+-----------\n";
+
+ for (const auto &util : utility_list) {
+ /* TODO(shivansh) Check before overwriting existing test scripts. */
+ test_file = tests_dir + util.first + "_test.sh";
+ std::cout << std::setw(21) << util.first + '(' + util.second + ") | ";
+ fflush(stdout); /* Useful in debugging. */
+ generatetest::GenerateTest(util.first, util.second, license);
+ std::cout << "Done!\n";
+ }
+
+ return EXIT_SUCCESS;
+}
Index: tools/tools/smoketestsuite/read_annotations.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/read_annotations.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <unordered_set>
+
+namespace annotations {
+ void read_annotations(std::string, \
+ std::unordered_set<std::string>&);
+}
Index: tools/tools/smoketestsuite/read_annotations.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/read_annotations.cpp
@@ -0,0 +1,63 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dirent.h>
+#include <fstream>
+#include <iostream>
+#include <string>
+
+#include "read_annotations.h"
+
+/*
+ * Read the annotation files and skip
+ * generation of the respective tests.
+ */
+void
+annotations::read_annotations(std::string utility,
+ std::unordered_set<std::string>& annotation_set)
+{
+ std::string line;
+ std::ifstream annotation_fstream;
+ annotation_fstream.open("annotation_set/" + utility + "_test.annot");
+
+ while (getline(annotation_fstream, line)) {
+ /* Add a unique identifier for no_arguments testcase */
+ if (!line.compare(0, 12, "no_arguments"))
+ annotation_set.insert("*");
+
+ /*
+ * Add flag value for supported argument testcases
+ * Doing so we ignore the "invalid_usage" testcase
+ * as it is guaranteed to always succeed.
+ */
+ else if (!line.compare(2, 4, "flag"))
+ annotation_set.insert(line.substr(0, 1));
+ }
+
+ annotation_fstream.close();
+}
Index: tools/tools/smoketestsuite/scripts/README
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/README
@@ -0,0 +1,8 @@
+# Helper scripts
+
+Script Name | Functionality
+------------------+-----------------
+fetch_groff.sh | Lists location of all section 1 utilities
+fetch_utils.sh | Saves all the base utilities in the src tree in **utils_list**
+generate_annot.sh | Populates annotation files under [annotations](../annotations)
+validate.sh | Validates side-effects of newly introduced changes in the tool
Index: tools/tools/smoketestsuite/scripts/fetch_groff.sh
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/fetch_groff.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# Copyright 2017 Shivansh Rai
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# Script for listing location of groff scripts of utilities in section 1
+
+set -e
+
+src="../../../"
+dir_list="tools/tools/smoketestsuite/scripts/utils_list"
+groff_src="tools/tools/smoketestsuite/groff"
+
+cd "$src"
+rm -rf "$groff_src" && mkdir "$groff_src"
+
+while IFS= read -r dir_entry
+do
+ for file in "$dir_entry"/*
+ do
+ case "$file" in
+ *.1) cp "$file" "$groff_src" ;;
+ *.8) cp "$file" "$groff_src" ;;
+ esac
+ done
+done< "$dir_list"
Index: tools/tools/smoketestsuite/scripts/fetch_utils.sh
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/fetch_utils.sh
@@ -0,0 +1,43 @@
+#! /bin/sh
+#
+# Copyright 2017 Shivansh Rai
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# Script for listing all the base utilities
+
+set -e
+script_dir="scripts"
+src="../../../"
+
+fetch_utils() {
+ cd "$src"
+ find . -name Makefile | xargs grep -l 'PROG\|PROG_CXX' \
+ | sed -e 's|/Makefile$||' | cut -c 3-
+}
+
+rm -f "$script_dir/utils_list"
+
+(fetch_utils) >> "$script_dir/utils_list"
Index: tools/tools/smoketestsuite/scripts/generate_annot.sh
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/generate_annot.sh
@@ -0,0 +1,91 @@
+#!/bin/sh
+#
+# Copyright 2017 Shivansh Rai
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# Script for generating annotations based on generated tests.
+
+suffix="_test"
+extension=".sh"
+update_required=0
+
+message="Following annotation files were updated. \n"
+
+printf "
++--------------------------------------------------+
+| Checking if any annotation file is to be updated |
++--------------------------------------------------+\n"
+
+for f in "generated_tests"/*
+do
+ annotations=""
+ file=$(basename "$f")
+ test=${file%$extension}
+ utility=${test%$suffix}
+ test_dir="/usr/tests/bin/$utility"
+
+ (
+ cd "$test_dir" || exit
+ report=$(kyua report)
+ i=2
+
+ while true
+ do
+ testcase=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $1}')
+ status=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $3}')
+ check=$(printf "%s" "$testcase" | cut -s -f1 -d":")
+
+ if [ "$check" != "$test" ]; then
+ if [ "$annotations" ]; then
+ if [ $update_required = 0 ]; then
+ printf $message
+ update_required=1
+ fi
+
+ annotations_file="annotations/$test.annot"
+ # Append only the new annotations
+ printf "$annotations" > "$annotations_file.temp"
+ [ ! -e "$annotations_file" ] && touch "$annotations_file"
+ comm -13 "$annotations_file" "$annotations_file.temp" >> \
+ "$annotations_file" && printf "\t%s\n" "annotations/$test.annot"
+ rm -f "$annotations_file.temp"
+ fi
+
+ break
+ fi
+
+ if [ "$status" = "failed:" ]; then
+ testcase=${testcase#"$test:"}
+ annotations="$annotations$testcase\n"
+ fi
+
+ i=$((i+1))
+ done
+ )
+
+done
+
+printf "==============================================================================\n\n"
Index: tools/tools/smoketestsuite/utils.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/utils.h
@@ -0,0 +1,72 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <list>
+#include <unordered_map>
+
+namespace utils {
+ /*
+ * Option relation which maps option names to
+ * a unique identifier in their description.
+ */
+ typedef struct OptRelation {
+ char type; /* Option type: (s)short/(l)long. */
+ std::string value; /* Name of the option. */
+ std::string keyword; /* The keyword which should be looked up in usage
+ * message (if) produced when using this option.
+ */
+ } opt_relation;
+
+ /*
+ * Read/Write file descriptors for a pipe.
+ */
+ struct PipeDescriptor {
+ int readfd;
+ int writefd;
+ pid_t pid; // PID of the forked child.
+ };
+
+ std::pair<std::string, int> Execute(std::string);
+ PipeDescriptor* POpen(const char*);
+
+ class OptDefinition {
+ public:
+ std::list<std::string> opt_list; /* list of all the accepted options
+ * with unknown usage.
+ */
+ std::unordered_map<std::string,
+ opt_relation> opt_map; /* Map "option value" to
+ * "option definition".
+ */
+ std::unordered_map<std::string,
+ opt_relation>::iterator opt_map_iter;
+
+ void InsertOpts();
+ std::list<opt_relation *> CheckOpts(std::string);
+ };
+}
Index: tools/tools/smoketestsuite/utils.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/utils.cpp
@@ -0,0 +1,306 @@
+/*-
+ * Copyright 2017 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <array>
+#include <cstdlib>
+#include <fcntl.h>
+#include <fstream>
+#include <iostream>
+#include <pthread.h>
+#include <signal.h>
+#include <string.h>
+#include <sys/select.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "utils.h"
+
+#define READ 0 /* Pipe descriptor: read end. */
+#define WRITE 1 /* Pipe descriptor: write end. */
+#define BUFSIZE 128 /* Buffer size (used for buffering output
+ * from a utility's execution).
+ */
+#define TIMEOUT 2 /* threshold (seconds) for a function call to return. */
+
+/*
+ * Insert a list of user-defined option definitions
+ * into a hashmap. These specific option definitions
+ * are the ones which one can be easily tested.
+ */
+void
+utils::OptDefinition::InsertOpts()
+{
+ /* Option definitions. */
+ OptRelation h_def; /* '-h' */
+ h_def.type = 's';
+ h_def.value = "h";
+ h_def.keyword = "help";
+
+ OptRelation v_def; /* '-v' */
+ v_def.type = 's';
+ v_def.value = "v";
+ v_def.keyword = "version";
+
+ /*
+ * "opt_map" contains all the options
+ * which can be easily tested.
+ */
+ opt_map.insert(std::make_pair<std::string, OptRelation>
+ ("h", (OptRelation)h_def));
+ opt_map.insert(std::make_pair<std::string, OptRelation>
+ ("v", (OptRelation)v_def));
+};
+
+/*
+ * For the utility under test, find the supported options
+ * present in the hashmap generated by InsertOpts()
+ * and return them in a form of list of option relations.
+ */
+std::list<utils::OptRelation *>
+utils::OptDefinition::CheckOpts(std::string utility)
+{
+ std::string line; /* An individual line in a man-page. */
+ std::string opt_name; /* Name of the option. */
+ std::string opt_identifier = ".It Fl"; /* Identifier for an option in man page. */
+ std::string buffer; /* Option description extracted from man-page. */
+ std::string opt_string; /* Identified option names. */
+ int opt_position; /* Starting index of the (identified) option. */
+ int space_index; /* First occurrence of space character
+ * in a multi-word option definition.
+ */
+ std::list<OptRelation *> identified_opt_list; /* List of identified option relations. */
+ std::list<std::string> supported_sections = { "1", "8" };
+
+ /* Generate the hashmap opt_map. */
+ InsertOpts();
+
+ for (const auto &section : supported_sections) {
+ std::ifstream infile("groff/" + utility + "." + section);
+
+ /*
+ * Search for all the options accepted by the
+ * utility and collect those present in "opt_map".
+ */
+ while (std::getline(infile, line)) {
+ if ((opt_position = line.find(opt_identifier)) != std::string::npos) {
+ /* Locate the position of option name. */
+ opt_position += opt_identifier.length() + 1;
+
+ if (opt_position > line.length()) {
+ /*
+ * This condition will trigger when a utility
+ * supports an empty argument, e.g. tset(issue #9)
+ */
+ continue;
+ }
+
+ /*
+ * Check for long options ; While here, also sanitize
+ * multi-word option definitions in a man page to properly
+ * extract short options from option definitions such as:
+ * .It Fl r Ar seconds (taken from date(1)).
+ */
+ if ((space_index = line.find(" ", opt_position + 1, 1))
+ != std::string::npos)
+ opt_name = line.substr(opt_position, space_index - opt_position);
+ else
+ opt_name = line.substr(opt_position);
+
+ /*
+ * Check if the identified option matches the identifier.
+ * "opt_list.back()" is the previously checked option, the
+ * description of which is now stored in "buffer".
+ */
+ if (!opt_list.empty() &&
+ (opt_map_iter = opt_map.find(opt_list.back()))
+ != opt_map.end() &&
+ buffer.find((opt_map_iter->second).keyword) != std::string::npos) {
+ identified_opt_list.push_back(&(opt_map_iter->second));
+
+ /*
+ * Since the usage of the option under test
+ * is known, we remove it from "opt_list".
+ */
+ opt_list.pop_back();
+ }
+
+ /* Update the list of valid options. */
+ opt_list.push_back(opt_name);
+
+ /* Empty the buffer for next option's description. */
+ buffer.clear();
+ } else {
+ /*
+ * Collect the option description until next
+ * valid option definition is encountered.
+ */
+ buffer.append(line);
+ }
+ }
+ }
+
+ return identified_opt_list;
+}
+
+/*
+ * When pclose() is called on the stream returned by popen(),
+ * it waits indefinitely for the created shell process to
+ * terminate in cases where the shell command waits for the
+ * user input via a blocking read (e.g. passwd(1)).
+ * Hence, we define a custom function which alongside
+ * returning the read-write file descriptors, also returns
+ * the pid of the newly created (child) shell process. This
+ * pid can later be used for sending a signal to the child.
+ *
+ * NOTE For the sake of completeness, we also specify actions
+ * to be taken corresponding to the "w" (write) type. However,
+ * in this context only the "r" (read) type will be required.
+ */
+
+utils::PipeDescriptor*
+utils::POpen(const char *command)
+{
+ int pdes[2];
+ char *argv[4];
+ pid_t child_pid;
+ PipeDescriptor *descr = (PipeDescriptor *)malloc(sizeof(PipeDescriptor));
+
+ /* Create a pipe with ~
+ * - pdes[READ]: read end
+ * - pdes[WRITE]: write end
+ */
+ if (pipe2(pdes, O_CLOEXEC) < 0)
+ return NULL;
+
+ descr->readfd = pdes[READ];
+ descr->writefd = pdes[WRITE];
+
+ argv[0] = (char *)"sh"; /* Type-cast to avoid compiler warning [-Wwrite-strings]. */
+ argv[1] = (char *)"-c";
+ argv[2] = (char *)command;
+ argv[3] = NULL;
+
+ switch (child_pid = vfork()) {
+ case -1: /* Error. */
+ return NULL;
+ case 0: /* Child. */
+ // TODO Verify if the following operations on both read
+ // and write file-descriptors (irrespective of the value
+ // of 'type', which is no longer being used) is safe.
+ if (pdes[WRITE] != STDOUT_FILENO)
+ dup2(pdes[WRITE], STDOUT_FILENO);
+ else
+ fcntl(pdes[WRITE], F_SETFD, 0);
+ if (pdes[READ] != STDIN_FILENO)
+ dup2(pdes[READ], STDIN_FILENO);
+ else
+ fcntl(pdes[READ], F_SETFD, 0);
+
+ /*
+ * For current usecase, it might so happen that the child gets
+ * stuck on a blocking read (e.g. passwd(1)) waiting for user
+ * input. In that case the child will be killed via a signal.
+ * To avoid any effect on the parent's execution, we place the
+ * child in a separate process group with pgid set as "child_pid".
+ */
+ setpgid(child_pid, child_pid);
+ execve("/bin/sh", argv, NULL);
+ exit(127);
+ }
+
+ descr->pid = child_pid;
+ return descr;
+}
+
+/*
+ * Executes the passed argument "command" in a shell
+ * and returns its output and the exit status.
+ */
+std::pair<std::string, int>
+utils::Execute(std::string command)
+{
+ int result;
+ std::array<char, BUFSIZE> buffer;
+ std::string usage_output;
+ struct timeval tv;
+ fd_set readfds;
+ FILE *pipe;
+ PipeDescriptor *pipe_descr;
+
+ pipe_descr = utils::POpen(command.c_str());
+ if (pipe_descr == NULL) {
+ perror("utils::POpen()");
+ exit(EXIT_FAILURE);
+ }
+
+ // Close the unrequired file-descriptor.
+ close(pipe_descr->writefd);
+
+ pipe = fdopen(pipe_descr->readfd, "r");
+ if (pipe == NULL) {
+ close(pipe_descr->readfd);
+ perror("fdopen()");
+ exit(EXIT_FAILURE);
+ }
+
+ /*
+ * Set a timeout value for the spawned shell
+ * process to complete its execution.
+ */
+ tv.tv_sec = TIMEOUT;
+ tv.tv_usec = 0;
+
+ FD_ZERO(&readfds);
+ FD_SET(fileno(pipe), &readfds);
+ result = select(fileno(pipe) + 1, &readfds, NULL, NULL, &tv);
+
+ if (result > 0) {
+ while (!feof(pipe))
+ if (fgets(buffer.data(), BUFSIZE, pipe) != NULL)
+ usage_output += buffer.data();
+
+ } else if (result == -1) {
+ perror("select()");
+ kill(pipe_descr->pid, SIGTERM);
+ exit(EXIT_FAILURE);
+ }
+
+ /*
+ * We gave a relaxed value of 2 seconds for the shell process
+ * to complete it's execution. If at this point it is still
+ * alive, it (most probably) is stuck on a blocking read
+ * waiting for the user input. Since few of the utilities
+ * performing such blocking reads don't respond to SIGINT
+ * (e.g. pax(1)), we terminate the shell process via SIGTERM.
+ */
+ if (kill(pipe_descr->pid, SIGTERM) < 0)
+ perror("kill()");
+
+ return std::make_pair<std::string, int>
+ ((std::string)usage_output, WEXITSTATUS(pclose(pipe)));
+}

File Metadata

Mime Type
text/plain
Expires
Wed, Nov 26, 6:16 PM (13 h, 43 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
26221871
Default Alt Text
D12249.id35835.diff (52 KB)

Event Timeline