Page MenuHomeFreeBSD
Files F166822895

D12249.id37452.diff
No OneTemporary
Actions

D12249.id37452.diff
View Options

Index: tools/tools/smoketestsuite/Makefile
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/Makefile
@@ -0,0 +1,31 @@
+# $FreeBSD$
+#
+# Makefile for building the test generation tool
+
+PROG_CXX= generate_tests
+LOCALBASE= /usr/local
+MAN=
+CXXFLAGS+= -I${LOCALBASE}/include -std=c++11
+LDFLAGS+= -L${LOCALBASE}/lib -lboost_filesystem -lboost_system
+SRCS= logging.cpp \
+ utils.cpp \
+ read_annotations.cpp \
+ generate_license.cpp \
+ add_testcase.cpp \
+ fetch_groff.cpp \
+ generate_test.cpp
+
+.PHONY: clean \
+ fetch_utils \
+ run
+
+fetch_utils:
+ sh ${.CURDIR}/scripts/fetch_utils.sh
+
+run:
+ @echo Generating annotations...
+ sh ${.CURDIR}/scripts/generate_annot.sh
+ @echo Generating test files...
+ ./generate_tests
+
+.include <bsd.prog.mk>
Index: tools/tools/smoketestsuite/README
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/README
@@ -0,0 +1,50 @@
+$FreeBSD$
+
+Smoke testing of base utilities
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Test generation tool made as a part of Google Summer of Code '17 with FreeBSD (https://summerofcode.withgoogle.com/projects/#6426676740227072).
+Refer the FreeBSD wiki (https://wiki.freebsd.org/SummerOfCode2017/SmokeTestingOfBaseUtilities) for an overview and updates.
+A brief description and implementation details: https://lists.freebsd.org/pipermail/soc-status/2017-July/001079.html.
+The diagram "architecture.png" briefly summarizes how different components fit with the testcase-generator.
+
+Project layout
+~~~~~~~~~~~~~~
+
+.
+├── annotations
+│   └── ........................:: Annotation files (generated/user-defined)
+├── scripts
+│   └── ........................:: Helper scripts
+├── architecture.png ...........:: A brief architecture diagram
+├── add_testcase.cpp ...........:: Testcase generator
+├── generate_license.cpp .......:: Customized license generator
+├── generate_test.cpp ..........:: Test generator
+├── logging.cpp ................:: Logger
+├── read_annotations.cpp .......:: Annotation parser
+└── utils.cpp ..................:: Index generator
+
+- - -
+
+Dependencies
+~~~~~~~~~~~~
+* Boost C++ libraries: The tool was tested to work with the port "boost-all-1.64.0".
+* Kyua
+
+Instructions
+~~~~~~~~~~~~
+* The tool needs to know about the utilities in src which don't already have tests. The list of all such utilities can be generated via -
+
+ make fetch_utils
+
+ This will generate the file "scripts/utils_list".
+
+* For generating the tests, execute the following -
+
+ make clean
+ make && make run
+
+ToDo
+~~~~
+The following features/functionalities are planned to be integrated -
+* [Batch mode] Appropriately update `BSD.tests.dist`.
Index: tools/tools/smoketestsuite/add_testcase.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/add_testcase.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _ADD_TESTCASE_H_
+#define _ADD_TESTCASE_H_
+
+namespace addtestcase {
+ void KnownTestcase(std::string, std::string, std::string, \
+ std::string, std::ofstream&);
+
+ void UnknownTestcase(std::string, std::string, std::pair<std::string, int>, \
+ std::string&, bool);
+
+ void NoArgsTestcase(std::string, std::pair<std::string, int>, \
+ std::ofstream&, bool);
+}
+
+#endif /* _ADD_TESTCASE_H_ */
Index: tools/tools/smoketestsuite/add_testcase.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/add_testcase.cpp
@@ -0,0 +1,177 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <fstream>
+#include <iostream>
+
+#include "add_testcase.h"
+
+/* Adds a test-case for an option with known usage. */
+void
+addtestcase::KnownTestcase(std::string option,
+ std::string util_with_section,
+ std::string descr,
+ std::string output,
+ std::ofstream& test_script)
+{
+ std::string testcase_name;
+ std::string utility = util_with_section.substr(0,
+ util_with_section.size() - 3);
+
+ /* Add testcase name. */
+ test_script << "atf_test_case ";
+ if (!option.empty()) {
+ testcase_name = option;
+ testcase_name.append("_flag");
+ } else
+ testcase_name = "no_arguments";
+ test_script << testcase_name + "\n";
+
+ /* Add testcase description. */
+ test_script << testcase_name
+ + "_head()\n{\n\tatf_set \"descr\" ";
+ if (!descr.empty())
+ test_script << descr;
+ else
+ test_script << "\"Verify the usage of option \'"
+ + option + "\'\"";
+ test_script << "\n}\n\n";
+
+ /* Add body of the testcase. */
+ test_script << testcase_name + "_body()\n{"
+ + "\n\tatf_check -s exit:0 -o ";
+
+ /* Match the usage output if generated. */
+ if (!output.empty())
+ test_script << "inline:\"" + output + "\" ";
+ else
+ test_script << "empty ";
+ test_script << utility;
+
+ if (!option.empty())
+ test_script << " -" + option;
+ test_script << "\n}\n\n";
+}
+
+/* Adds a test-case for an option with unknown usage. */
+void
+addtestcase::UnknownTestcase(std::string option,
+ std::string util_with_section,
+ std::pair<std::string, int> output,
+ std::string& testcase_buffer,
+ bool usage_output)
+{
+ std::string utility = util_with_section.substr(0,
+ util_with_section.size() - 3);
+
+ if (output.second)
+ testcase_buffer.append("\n\tatf_check -s not-exit:0 -e ");
+ else
+ testcase_buffer.append("\n\tatf_check -s exit:0 -o ");
+
+ /* Check if a usage message was produced (case-insensitive match). */
+ if (usage_output)
+ testcase_buffer.append("match:\"$usage_output\" ");
+ else if (!output.first.empty())
+ testcase_buffer.append("inline:\"" + output.first + "\" ");
+ else
+ testcase_buffer.append("empty ");
+
+ testcase_buffer.append(utility);
+
+ if (!option.empty())
+ testcase_buffer.append(" -" + option);
+}
+
+/* Adds a test-case for usage without any arguments. */
+void
+addtestcase::NoArgsTestcase(std::string util_with_section,
+ std::pair<std::string, int> output,
+ std::ofstream& test_script,
+ bool usage_output)
+{
+ std::string descr;
+ std::string utility = util_with_section.substr(0,
+ util_with_section.size() - 3);
+
+ if (output.second) {
+ /* An error was encountered. */
+ test_script << std::string("atf_test_case no_arguments\n")
+ + "no_arguments_head()\n{\n\tatf_set \"descr\" ";
+ if (!output.first.empty()) {
+ /*
+ * We expect a usage message to be generated in this
+ * case (case-insensitive match).
+ */
+ if (usage_output) {
+ descr = "\"Verify that " + util_with_section
+ + " fails and generates a valid usage \" "
+ + "\\\n\t\t\t\"message when no arguments "
+ + "are supplied\"";
+
+ test_script << descr
+ + "\n}\n\nno_arguments_body()\n{"
+ + "\n\tatf_check -s not-exit:0 -e match:"
+ + "\"$usage_output\" " + utility;
+ } else {
+ descr = "\"Verify that " + util_with_section
+ + " fails and generates a valid output \" "
+ + "\\\n\t\t\t\"when no arguments are supplied\"";
+
+ test_script << descr
+ + "\n}\n\nno_arguments_body()\n{"
+ + "\n\tatf_check -s not-exit:0 -e inline:"
+ + "\"" + output.first + "\" "
+ + utility;
+ }
+ } else {
+ descr = "\"Verify that " + util_with_section + " fails "
+ + "silently when no arguments are supplied\"" ;
+ test_script << descr + "\n}\n\nno_arguments_body()\n{"
+ + "\n\tatf_check -s not-exit:0 -e empty "
+ + utility;
+ }
+ test_script << "\n}\n\n";
+ } else {
+ /*
+ * The command ran successfully, hence we guessed
+ * a correct usage for the utility under test.
+ */
+ if (!output.first.empty())
+ descr = "\"Verify that " + util_with_section + " executes "
+ + "successfully and produces a valid \" \\\n\t\t\t"
+ + "\"output when invoked without any arguments\"";
+ else
+ descr = "\"Verify that " + util_with_section
+ + " executes successfully and silently \" \\\n"
+ + "\t\t\t\"when invoked without any arguments\"";
+
+ addtestcase::KnownTestcase("", util_with_section, descr,
+ output.first, test_script);
+ }
+}
Index: tools/tools/smoketestsuite/annotations/date_test.ant
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/annotations/date_test.ant
@@ -0,0 +1,5 @@
+R_flag
+j_flag
+n_flag
+no_arguments
+u_flag
Index: tools/tools/smoketestsuite/fetch_groff.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/fetch_groff.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _FETCH_GROFF_H_
+#define _FETCH_GROFF_H_
+
+#include <unordered_map>
+
+namespace groff {
+ extern std::unordered_map<std::string, std::string> groff_map;
+ int FetchGroffScripts();
+}
+
+#endif /* _FETCH_GROFF_H_ */
Index: tools/tools/smoketestsuite/fetch_groff.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/fetch_groff.cpp
@@ -0,0 +1,98 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dirent.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <boost/filesystem.hpp>
+#include <boost/thread.hpp>
+#include <fstream>
+#include <iostream>
+#include <regex>
+
+#include "fetch_groff.h"
+#include "logging.h"
+
+/* Map of utility name and its location in src tree. */
+std::unordered_map<std::string, std::string> groff::groff_map;
+
+/*
+ * Traverses the FreeBSD src tree looking for groff scripts for section
+ * 1 and section 8 utilities and stores their location in a hashmap.
+ */
+int
+groff::FetchGroffScripts()
+{
+ std::string utils_list = "scripts/utils_list";
+ std::string src = "../../../"; /* FreeBSD src. */
+ std::string utildir;
+ std::string utilname;
+ std::string path;
+ std::ifstream file;
+ std::regex section ("(.*).(?:1|8)");
+ struct stat sb;
+ struct dirent *ent;
+ DIR *dir;
+
+ /* Check if the file "scripts/utils_list" exists. */
+ if (stat(utils_list.c_str(), &sb) != 0) {
+ std::cerr << "scripts/utils_list does not exists.\n"
+ "Run 'make fetch_utils' first.\n";
+ return EXIT_FAILURE;
+ }
+ file.open(utils_list);
+
+ while (getline(file, utildir)) {
+ path = src + utildir + "/tests";
+ /*
+ * Copy the groff script only if the utility does not
+ * already have tests, i.e. the "tests" directory is absent.
+ */
+ if (stat(path.c_str(), &sb) || !S_ISDIR(sb.st_mode)) {
+ path = src + utildir + "/";
+ utilname = utildir.substr(utildir.find_last_of("/") + 1);
+ if ((dir = opendir(path.c_str())) != NULL) {
+ /* Skip directory entry for "." and "..". */
+ ent = readdir(dir);
+ ent = readdir(dir);
+ while ((ent = readdir(dir)) != NULL) {
+ if (std::regex_match(ent->d_name, section)) {
+ groff_map[utilname] = path + ent->d_name;
+ }
+ }
+ closedir(dir);
+ } else {
+ logging::LogPerror("opendir()");
+ }
+ }
+ }
+
+ file.close();
+ return EXIT_SUCCESS;
+}
Index: tools/tools/smoketestsuite/generate_license.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_license.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _GENERATE_LICENSE_H_
+#define _GENERATE_LICENSE_H_
+
+namespace generatelicense {
+ std::string GenerateLicense(int, char **);
+}
+
+#endif /* _GENERATE_LICENSE_H_ */
Index: tools/tools/smoketestsuite/generate_license.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_license.cpp
@@ -0,0 +1,82 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+
+#include "generate_license.h"
+#include "utils.h"
+
+std::string
+generatelicense::GenerateLicense(int argc, char **argv)
+{
+ std::string license;
+ std::string copyright_owner;
+
+ if (argc > 1) {
+ if (argc == 3 && strncmp(argv[1], "--name ", 7))
+ copyright_owner = argv[2];
+ else {
+ std::cerr << "Usage: ./generate_tests --name <copyright_owner>\n";
+ exit(EXIT_FAILURE);
+ }
+ } else
+ copyright_owner = utils::Execute("id -P | cut -d : -f 8").first;
+
+ license =
+ "#\n"
+ "# Copyright 2017-2018 " + copyright_owner + "\n"
+ "# All rights reserved.\n"
+ "#\n"
+ "# Redistribution and use in source and binary forms, with or without\n"
+ "# modification, are permitted provided that the following conditions\n"
+ "# are met:\n"
+ "# 1. Redistributions of source code must retain the above copyright\n"
+ "# notice, this list of conditions and the following disclaimer.\n"
+ "# 2. Redistributions in binary form must reproduce the above copyright\n"
+ "# notice, this list of conditions and the following disclaimer in the\n"
+ "# documentation and/or other materials provided with the distribution.\n"
+ "#\n"
+ "# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND\n"
+ "# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n"
+ "# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n"
+ "# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n"
+ "# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n"
+ "# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n"
+ "# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n"
+ "# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n"
+ "# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n"
+ "# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"
+ "# SUCH DAMAGE.\n"
+ "#\n"
+ "# $FreeBSD$\n"
+ "#\n\n";
+
+ return license;
+}
Index: tools/tools/smoketestsuite/generate_test.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_test.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _GENERATE_TEST_H_
+#define _GENERATE_TEST_H_
+
+#include "utils.h"
+
+namespace generatetest {
+ void IntHandler(int);
+ void GenerateMakefile(std::string, std::string);
+ void GenerateTest(std::string, char,
+ std::string&, const char*);
+}
+
+#endif /* _GENERATE_TEST_H_ */
Index: tools/tools/smoketestsuite/generate_test.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/generate_test.cpp
@@ -0,0 +1,404 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dirent.h>
+#include <signal.h>
+#include <sys/stat.h>
+
+#include <boost/algorithm/string.hpp>
+#include <boost/filesystem.hpp>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <unordered_set>
+
+#include "add_testcase.h"
+#include "fetch_groff.h"
+#include "generate_license.h"
+#include "generate_test.h"
+#include "logging.h"
+#include "read_annotations.h"
+
+void
+generatetest::IntHandler(int dummmy)
+{
+ std::cerr << "\nExiting...\n";
+ /* Remove the temporary directory. */
+ boost::filesystem::remove_all(utils::tmpdir);
+ exit(EXIT_FAILURE);
+}
+
+/* [Batch mode] Generate a makefile for the test of given utility. */
+void
+generatetest::GenerateMakefile(std::string utility, std::string utildir)
+{
+ std::ofstream file;
+
+ file.open(utildir + "/Makefile", std::ios::out);
+ file << "# $FreeBSD$\n\nATF_TESTS_SH+= "
+ + utility + "_test\n\n"
+ + ".include <bsd.test.mk>\n";
+ file.close();
+}
+
+/* Generate a test for the given utility. */
+void
+generatetest::GenerateTest(std::string utility,
+ char section,
+ std::string& license,
+ const char *testsdir)
+{
+ std::vector<std::string> usage_messages;
+ std::vector<utils::OptRelation *> identified_opts;
+ std::string command;
+ std::string testcase_list;
+ std::string buffer;
+ std::string testfile;
+ std::string util_with_section;
+ std::ofstream file;
+ std::pair<std::string, int> output;
+ std::unordered_set<std::string> annotation_set;
+ /* Number of options for which a testcase has been generated. */
+ int progress = 0;
+ bool usage_output = false; /* Tracks whether '$usage_output' variable is used. */
+
+ /* Read annotations and populate hash set "annotation_set". */
+ annotations::read_annotations(utility, annotation_set);
+ util_with_section = utility + '(' + section + ')';
+ utils::OptDefinition opt_def;
+ identified_opts = opt_def.CheckOpts(utility);
+ testfile = testsdir + utility + "_test.sh";
+
+#ifndef DEBUG
+ /* Indicate the start of test generation for current utility. */
+ if (isatty(fileno(stderr))) {
+ std::cerr << std::setw(18) << util_with_section << " | "
+ << progress << "/" << opt_def.opt_list.size() << "\r";
+ }
+#endif
+ /* Add license in the generated test scripts. */
+ file.open(testfile, std::ios::out);
+ file << license;
+
+ /*
+ * If a known option was encountered (i.e. `identified_opts` is
+ * populated), produce a testcase to check the validity of the
+ * result of that option. If no known option was encountered,
+ * produce testcases to verify the correct (generated) usage
+ * message when using the supported options incorrectly.
+ */
+ for (const auto &i : identified_opts) {
+ command = utils::GenerateCommand(utility, i->value);
+ output = utils::Execute(command);
+ if (boost::iequals(output.first.substr(0, 6), "usage:")) {
+ /* Our guessed usage is incorrect as usage message is produced. */
+ addtestcase::UnknownTestcase(i->value, util_with_section,
+ output, buffer, usage_output);
+ } else {
+ addtestcase::KnownTestcase(i->value, util_with_section,
+ "", output.first, file);
+ }
+ testcase_list.append("\tatf_add_test_case " + i->value + "_flag\n");
+ }
+
+ /* Add testcases for the options whose usage is not yet known.
+ * For the purpose of adding a "$usage_output" variable,
+ * we choose the option which produces one.
+ * TODO Avoid double executions of an option, i.e. one while
+ * selecting usage message and another while generating testcase.
+ */
+ if (opt_def.opt_list.size() == 1) {
+ /* Check if the single option produces a usage message. */
+ command = utils::GenerateCommand(utility, opt_def.opt_list.front());
+ output = utils::Execute(command);
+ if (output.second && !output.first.empty()) {
+ usage_output = true;
+ file << "usage_output=\'" + output.first + "\'\n\n";
+ }
+ } else if (opt_def.opt_list.size() > 1) {
+ /*
+ * Utility supports multiple options. In case the usage message
+ * is consistent for atleast "two" options, we reduce duplication
+ * by assigning a variable "usage_output" in the test script.
+ */
+ for (const auto &i : opt_def.opt_list) {
+ command = utils::GenerateCommand(utility, i);
+ output = utils::Execute(command);
+ if (output.second && usage_messages.size() < 3)
+ usage_messages.push_back(output.first);
+ }
+
+ for (int j = 0; j < usage_messages.size(); j++) {
+ if (!usage_messages[j].compare
+ (usage_messages[(j+1) % usage_messages.size()])) {
+ usage_output = true;
+ file << "usage_output=\'"
+ + output.first.substr(0, 7 + utility.size())
+ + "\'\n\n";
+ break;
+ }
+ }
+ }
+
+ /*
+ * Execute the utility with supported options, while
+ * adding positive and negative testcases accordingly.
+ */
+ for (const auto &i : opt_def.opt_list) {
+ /* Ignore the option if it is annotated. */
+ if (annotation_set.find(i) != annotation_set.end())
+ continue;
+
+ command = utils::GenerateCommand(utility, i);
+ output = utils::Execute(command);
+#ifndef DEBUG
+ if (isatty(fileno(stderr))) {
+ std::cerr << std::setw(18) << util_with_section
+ << " | " << ++progress << "/"
+ << opt_def.opt_list.size() << "\r";
+ }
+#endif
+ if (output.second) {
+ addtestcase::UnknownTestcase(i, util_with_section, output,
+ buffer, usage_output);
+ } else {
+ /* Guessed usage is correct as EXIT_SUCCESS is encountered */
+ addtestcase::KnownTestcase(i, util_with_section, "",
+ output.first, file);
+ testcase_list.append(std::string("\tatf_add_test_case ")
+ + i + "_flag\n");
+ }
+ }
+ std::cout << std::endl; /* Takes care of the last '\r'. */
+
+ if (!opt_def.opt_list.empty()) {
+ testcase_list.append("\tatf_add_test_case invalid_usage\n");
+ file << "atf_test_case invalid_usage\ninvalid_usage_head()\n"
+ << "{\n\tatf_set \"descr\" \"Verify that an invalid usage "
+ << "with a supported option \" \\\n\t\t\t\"produces a valid "
+ << "error message\"\n}\n\ninvalid_usage_body()\n{";
+
+ file << buffer + "\n}\n\n";
+ }
+
+ /*
+ * Add a testcase under "no_arguments" for
+ * running the utility without any arguments.
+ */
+ if (annotation_set.find("*") == annotation_set.end()) {
+ command = utils::GenerateCommand(utility, "");
+ output = utils::Execute(command);
+ addtestcase::NoArgsTestcase(util_with_section, output,
+ file, usage_output);
+ testcase_list.append("\tatf_add_test_case no_arguments\n");
+ }
+
+ file << "atf_init_test_cases()\n{\n" + testcase_list + "}\n";
+ file.close();
+}
+
+int
+main(int argc, char **argv)
+{
+ std::ifstream groff_list;
+ struct stat sb;
+ struct dirent *ent;
+ char answer;
+ std::string license;
+ std::string utildir; /* Path to utility in src tree. */
+ std::string groffpath;
+ const char *testsdir = "generated_tests/";
+ /*
+ * Instead of generating tests for all the utilities, "batch mode"
+ * allows generation of tests for first "batch_limit" number of
+ * utilities selected from "scripts/utils_list".
+ */
+ bool batch_mode = false;
+ int batch_limit; /* Number of tests to be generated in batch mode. */
+
+ /* Handle interrupts. */
+ signal(SIGINT, generatetest::IntHandler);
+
+ if (groff::FetchGroffScripts() == EXIT_FAILURE)
+ return EXIT_FAILURE;
+
+ /*
+ * Create a temporary directory where all the side-effects
+ * introduced by utility-specific commands are restricted.
+ */
+ boost::filesystem::create_directory(utils::tmpdir);
+
+ std::cout << "\nInstead of generating tests for all the utilities, 'batch mode'\n"
+ "allows generation of tests for first few utilities selected from\n"
+ "'scripts/utils_list', and places them at their correct location\n"
+ "in the src tree, with corresponding makefiles created.\n"
+ "NOTE: You will be prompted for the superuser password when\n"
+ "creating test directory under '/usr/tests/' and when installing\n"
+ "the tests via `sudo make install`.\n"
+ "Run in 'batch mode' ? [y/N] ";
+ std::cin.get(answer);
+
+ switch(answer) {
+ case 'y':
+ case 'Y':
+ batch_mode = true;
+ std::cout << "Number of utilities to select for test generation: ";
+ std::cin >> batch_limit;
+
+ if (batch_limit <= 0) {
+ std::cerr << "Invalid input. Exiting...\n";
+ return EXIT_FAILURE;
+ }
+ break;
+ case '\n':
+ default:
+ break;
+ }
+
+ /* Check if the directory "testsdir" exists. */
+ if (stat(testsdir, &sb) || !S_ISDIR(sb.st_mode)) {
+ boost::filesystem::path dir(testsdir);
+ if (boost::filesystem::create_directory(dir))
+ std::cout << "Directory created: " << testsdir << "\n";
+ else {
+ std::cerr << "Unable to create directory: " << testsdir << "\n";
+ return EXIT_FAILURE;
+ }
+ }
+
+ /* Generate a license to be added in the generated scripts. */
+ license = generatelicense::GenerateLicense(argc, argv);
+
+#ifndef DEBUG
+ /* Generate a tabular-like format. */
+ std::cout << std::endl;
+ std::cout << std::setw(30) << "Utility | Progress\n";
+ std::cout << std::setw(32) << "----------+-----------\n";
+#endif
+
+ if (batch_mode) {
+ /* Number of hops required to reach root directory. */
+ std::string hops = "../../../../";
+ std::string command;
+ std::string installdir; /* Directory where tests are installed. */
+ int retval;
+ int maxdepth = 32;
+ std::unordered_map<std::string, std::string>::iterator it;
+ /* Remember pwd so that we can return back. */
+ boost::filesystem::path tooldir = boost::filesystem::current_path();
+
+ /*
+ * Discover number of hops required to reach root directory.
+ * To avoid an infinite loop in case directory "usr/" doesn't
+ * exist in the filesystem, we assume that the maximum depth
+ * from root directory at which pwd is located is "maxdepth".
+ */
+ chdir(hops.c_str()); /* Move outside FreeBSD src. */
+ while (maxdepth--) {
+ if (chdir("..") == -1) {
+ perror("chdir");
+ return EXIT_FAILURE;
+ }
+ hops += "../";
+ if (stat("usr", &sb) == 0 && S_ISDIR(sb.st_mode))
+ break;
+ }
+
+ /*
+ * Generate tests for first "batch_limit" number of
+ * utilities selected from "scripts/utils_list".
+ */
+ it = groff::groff_map.begin();
+ while (batch_limit-- && it != groff::groff_map.end()) {
+ /* Move back to the tool's directory. */
+ boost::filesystem::current_path(tooldir);
+
+ groffpath = groff::groff_map.at(it->first);
+ utildir = groffpath.substr
+ (0, groffpath.size() - 2 - it->first.size());
+ installdir = hops + "usr/tests/" + utildir.substr(9);
+ utildir += "tests/";
+
+ /* Populate "tests/" directory. */
+ boost::filesystem::remove_all(utildir);
+ boost::filesystem::create_directory(utildir);
+ generatetest::GenerateMakefile(it->first, utildir);
+ generatetest::GenerateTest(it->first, it->second.back(),
+ license, utildir.c_str());
+ boost::filesystem::copy_file(utildir + it->first + "_test.sh",
+ testsdir + it->first + "_test.sh",
+ boost::filesystem::copy_option::overwrite_if_exists);
+ std::advance(it, 1);
+
+ /* Execute the generated test and note success/failure. */
+ if (stat(installdir.c_str(), &sb) || !S_ISDIR(sb.st_mode)) {
+ command = "sudo mkdir -p " + installdir;
+ if ((retval = system(command.c_str())) == -1) {
+ perror("system");
+ return EXIT_FAILURE;
+ } else if (retval) {
+ boost::filesystem::current_path(tooldir);
+ boost::filesystem::remove_all(utildir);
+ continue;
+ }
+ }
+
+ /* Install the test. */
+ chdir(utildir.c_str());
+ if ((retval = system("sudo make install")) == -1) {
+ perror("system");
+ return EXIT_FAILURE;
+ } else if (retval) {
+ boost::filesystem::current_path(tooldir);
+ boost::filesystem::remove_all(utildir);
+ continue;
+ }
+ boost::filesystem::current_path(tooldir);
+
+ /* Run the test. */
+ chdir(installdir.c_str());
+ if ((retval = system("kyua test")) == -1) {
+ perror("system");
+ return EXIT_FAILURE;
+ } else if (retval) {
+ boost::filesystem::current_path(tooldir);
+ boost::filesystem::remove_all(utildir);
+ continue;
+ }
+ }
+ } else {
+ for (const auto &it : groff::groff_map) {
+ generatetest::GenerateTest(it.first, it.second.back(),
+ license, testsdir);
+ }
+ }
+
+ /* Cleanup. */
+ boost::filesystem::remove_all(utils::tmpdir);
+ return EXIT_SUCCESS;
+}
Index: tools/tools/smoketestsuite/logging.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/logging.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _LOGGING_H_
+#define _LOGGING_H_
+
+/* Use a gcc variadic macro to conditionally compile debug printing. */
+#ifdef DEBUG
+#define DEBUGP(...) \
+ fprintf(stdout, __VA_ARGS__); \
+ fflush(stdout);
+#else
+#define DEBUGP(...) {}
+#endif /* DEBUG */
+
+namespace logging {
+ void LogPerror(const char *);
+}
+
+#endif /* _LOGGING_H_ */
Index: tools/tools/smoketestsuite/logging.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/logging.cpp
@@ -0,0 +1,39 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <iostream>
+
+#include "logging.h"
+
+void
+logging::LogPerror(const char *message)
+{
+#ifdef DEBUG
+ perror(message);
+#endif
+}
Index: tools/tools/smoketestsuite/read_annotations.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/read_annotations.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _READ_ANNOTATIONS_H_
+#define _READ_ANNOTATIONS_H_
+
+#include <unordered_set>
+
+namespace annotations {
+ void read_annotations(std::string, \
+ std::unordered_set<std::string>&);
+}
+
+#endif /* _READ_ANNOTATIONS_H_ */
Index: tools/tools/smoketestsuite/read_annotations.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/read_annotations.cpp
@@ -0,0 +1,58 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <fstream>
+#include <iostream>
+#include <string>
+
+#include "read_annotations.h"
+
+/* Read the annotation files and skip generation of respective tests. */
+void
+annotations::read_annotations(std::string utility,
+ std::unordered_set<std::string>& annotation_set)
+{
+ std::string line;
+ std::ifstream file;
+ file.open("annotations/" + utility + "_test.annot");
+
+ while (getline(file, line)) {
+ /* Add a unique identifier for no_arguments testcase */
+ if (!line.compare(0, 12, "no_arguments"))
+ annotation_set.insert("*");
+ /*
+ * Add flag value for supported argument testcases
+ * Doing so we ignore the "invalid_usage" testcase
+ * as it is guaranteed to always succeed.
+ */
+ else if (!line.compare(2, 4, "flag"))
+ annotation_set.insert(line.substr(0, 1));
+ }
+
+ file.close();
+}
Index: tools/tools/smoketestsuite/scripts/README
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/README
@@ -0,0 +1,9 @@
+Helper scripts
+~~~~~~~~~~~~~~
+
+Script Name | Functionality
+------------------+-----------------
+fetch_groff.sh | Lists location of all section 1 utilities
+fetch_utils.sh | Saves all the base utilities in the src tree in **utils_list**
+generate_annot.sh | Populates annotation files under [annotations](../annotations)
+validate.sh | Validates side-effects of newly introduced changes in the tool
Index: tools/tools/smoketestsuite/scripts/fetch_utils.sh
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/fetch_utils.sh
@@ -0,0 +1,44 @@
+#! /bin/sh
+#
+# Copyright 2017-2018 Shivansh Rai
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# Script for listing all the base utilities
+
+set -eu
+
+script_dir="$(dirname $0)"
+src="$(dirname $0)/../../../../"
+
+fetch_utils() {
+ cd "$src"
+ find . -name Makefile | xargs grep -l 'PROG\|PROG_CXX' \
+ | sed -e 's|/Makefile$||' | cut -c 3-
+}
+
+rm -f "$script_dir/utils_list"
+
+(fetch_utils) >> "$script_dir/utils_list"
Index: tools/tools/smoketestsuite/scripts/generate_annot.sh
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/generate_annot.sh
@@ -0,0 +1,107 @@
+#!/bin/sh
+#
+# Copyright 2017-2018 Shivansh Rai
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# Script for generating annotations based on generated tests.
+
+set -eu
+
+update_annotations() {
+ cd "$testdir" 2>/dev/null
+ report=$(kyua report)
+ cd -
+ i=2
+ annotations=""
+ diff=""
+
+ while true
+ do
+ testcase=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $1}')
+ exitstatus=$(printf "%s" "$report" | awk 'NR=='"$i"' {print $3}')
+ check=$(printf "%s" "$testcase" | cut -s -f1 -d":")
+
+ if [ ! "$check" ] && [ "$annotations" ]; then
+ file="$tooldir/annotations/$test.ant" # Annotations file
+ # Append only the new annotations.
+ printf "$annotations" > "$file.temp"
+ [ ! -e "$file" ] && touch "$file"
+ diff=$(comm -13 "$file" "$file.temp")
+ if [ "$diff" ]; then
+ if [ $prompt = 0 ]; then
+ prompt=1
+ printf "\nModified annotation files -\n"
+ fi
+ printf "$diff\n" >> "$file"
+ printf " * %s\n" "$test.ant"
+ fi
+ rm -f "$file.temp"
+ break
+ fi
+
+ if [ "$exitstatus" = "failed:" ]; then
+ testcase=${testcase#"$test:"}
+ annotations="$annotations$testcase\n"
+ elif [ ! "$exitstatus" ]; then
+ break
+ fi
+ i=$((i+1))
+ done
+}
+
+suffix="_test"
+extension=".sh"
+prompt=0
+tooldir=$(dirname $0)/..
+
+# Check if the directory "generated_tests/" is populated.
+if [ ! -d "$tooldir/generated_tests" ] || \
+ [ -z "$(ls -A $tooldir/generated_tests)" ]; then
+ exit
+fi
+
+for f in "$tooldir/generated_tests"/*
+do
+ file=$(basename "$f")
+ test=${file%$extension}
+ utility=${test%$suffix}
+
+ # Guess the correct installation directory.
+ if [ -d "/usr/tests/bin/$utility" ]; then
+ testdir="/usr/tests/bin/$utility"
+ elif [ -d "/usr/tests/usr.bin/$utility" ]; then
+ testdir="/usr/tests/usr.bin/$utility"
+ elif [ -d "/usr/tests/usr.sbin/$utility" ]; then
+ testdir="/usr/tests/usr.sbin/$utility"
+ else
+ continue
+ fi
+ update_annotations
+done
+
+if [ $prompt = 1 ]; then
+ printf "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n"
+fi
Index: tools/tools/smoketestsuite/scripts/utils_list
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/scripts/utils_list
@@ -0,0 +1,1251 @@
+
+bin/cat
+bin/chflags
+bin/chio
+bin/chmod
+bin/cp
+bin/csh
+bin/date
+bin/dd
+bin/df
+bin/domainname
+bin/echo
+bin/ed
+bin/expr
+bin/getfacl
+bin/hostname
+bin/kenv
+bin/kill
+bin/ln
+bin/ls
+bin/mkdir
+bin/mv
+bin/pax
+bin/pkill
+bin/ps
+bin/pwait
+bin/pwd
+bin/rcp
+bin/realpath
+bin/rm
+bin/rmail
+bin/rmdir
+bin/setfacl
+bin/sh
+bin/sleep
+bin/stty
+bin/sync
+bin/test
+bin/uuidgen
+cddl/sbin/zfs
+cddl/sbin/zpool
+cddl/usr.bin/ctfconvert
+cddl/usr.bin/ctfdump
+cddl/usr.bin/ctfmerge
+cddl/usr.bin/zinject
+cddl/usr.bin/zlook
+cddl/usr.bin/zstreamdump
+cddl/usr.bin/ztest
+cddl/usr.sbin/dtrace
+cddl/usr.sbin/lockstat
+cddl/usr.sbin/plockstat
+cddl/usr.sbin/zdb
+cddl/usr.sbin/zfsd
+cddl/usr.sbin/zhack
+contrib/blacklist/bin
+contrib/blacklist/test
+contrib/bmake
+contrib/elftoolchain/addr2line
+contrib/elftoolchain/ar
+contrib/elftoolchain/brandelf
+contrib/elftoolchain/cxxfilt
+contrib/elftoolchain/elfcopy
+contrib/elftoolchain/elfdump
+contrib/elftoolchain/nm
+contrib/elftoolchain/readelf
+contrib/elftoolchain/size
+contrib/elftoolchain/strings
+contrib/zstd/contrib/pzstd
+contrib/zstd/programs
+contrib/zstd/zlibWrapper
+contrib/ldns-host
+contrib/mdocml
+contrib/mtree
+contrib/ofed/libibverbs/examples/build/asyncwatch
+contrib/ofed/libibverbs/examples/build/device_list
+contrib/ofed/libibverbs/examples/build/devinfo
+contrib/ofed/libibverbs/examples/build/rc_pingpong
+contrib/ofed/libibverbs/examples/build/srq_pingpong
+contrib/ofed/libibverbs/examples/build/uc_pingpong
+contrib/ofed/libibverbs/examples/build/ud_pingpong
+contrib/ofed/librdmacm/examples/build/cmatose
+contrib/ofed/librdmacm/examples/build/mckey
+contrib/ofed/librdmacm/examples/build/rping
+contrib/ofed/librdmacm/examples/build/udaddy
+contrib/ofed/usr.bin/ibaddr
+contrib/ofed/usr.bin/ibnetdiscover
+contrib/ofed/usr.bin/ibping
+contrib/ofed/usr.bin/ibportstate
+contrib/ofed/usr.bin/ibroute
+contrib/ofed/usr.bin/ibsendtrap
+contrib/ofed/usr.bin/ibstat
+contrib/ofed/usr.bin/ibsysstat
+contrib/ofed/usr.bin/ibtracert
+contrib/ofed/usr.bin/opensm
+contrib/ofed/usr.bin/osmtest
+contrib/ofed/usr.bin/perfquery
+contrib/ofed/usr.bin/saquery
+contrib/ofed/usr.bin/sminfo
+contrib/ofed/usr.bin/smpdump
+contrib/ofed/usr.bin/smpquery
+contrib/ofed/usr.bin/vendstat
+contrib/pnpinfo
+contrib/smbfs/smbutil
+contrib/tzcode/zic/zdump
+contrib/tzcode/zic/zic
+contrib/unvis
+contrib/vis
+crypto/openssh/regress/misc/kexfuzz
+crypto/openssh/regress/unittests/bitmap
+crypto/openssh/regress/unittests/hostkeys
+crypto/openssh/regress/unittests/kex
+crypto/openssh/regress/unittests/match
+crypto/openssh/regress/unittests/sshbuf
+crypto/openssh/regress/unittests/sshkey
+crypto/openssh/regress/unittests/utf8
+crypto/openssh/regress/unittests/conversion
+crypto/openssl
+crypto/openssl/apps
+crypto/openssl/crypto
+crypto/openssl/crypto/aes
+crypto/openssl/crypto/asn1
+crypto/openssl/crypto/bf
+crypto/openssl/crypto/bio
+crypto/openssl/crypto/bn
+crypto/openssl/crypto/buffer
+crypto/openssl/crypto/camellia
+crypto/openssl/crypto/cast
+crypto/openssl/crypto/cmac
+crypto/openssl/crypto/cms
+crypto/openssl/crypto/des
+crypto/openssl/crypto/dh
+crypto/openssl/crypto/dsa
+crypto/openssl/crypto/dso
+crypto/openssl/crypto/ec
+crypto/openssl/crypto/ecdh
+crypto/openssl/crypto/ecdsa
+crypto/openssl/crypto/engine
+crypto/openssl/crypto/err
+crypto/openssl/crypto/hmac
+crypto/openssl/crypto/idea
+crypto/openssl/crypto/jpake
+crypto/openssl/crypto/lhash
+crypto/openssl/crypto/md2
+crypto/openssl/crypto/md4
+crypto/openssl/crypto/md5
+crypto/openssl/crypto/mdc2
+crypto/openssl/crypto/modes
+crypto/openssl/crypto/objects
+crypto/openssl/crypto/pkcs12
+crypto/openssl/crypto/pkcs7
+crypto/openssl/crypto/pqueue
+crypto/openssl/crypto/rand
+crypto/openssl/crypto/rc2
+crypto/openssl/crypto/rc4
+crypto/openssl/crypto/rc5
+crypto/openssl/crypto/ripemd
+crypto/openssl/crypto/rsa
+crypto/openssl/crypto/seed
+crypto/openssl/crypto/sha
+crypto/openssl/crypto/srp
+crypto/openssl/crypto/stack
+crypto/openssl/crypto/store
+crypto/openssl/crypto/ts
+crypto/openssl/crypto/txt_db
+crypto/openssl/crypto/ui
+crypto/openssl/crypto/whrlpool
+crypto/openssl/crypto/x509
+crypto/openssl/crypto/x509v3
+crypto/openssl/engines
+crypto/openssl/engines/ccgost
+crypto/openssl/ssl
+gnu/usr.bin/binutils/as
+gnu/usr.bin/binutils/ld
+gnu/usr.bin/binutils/objcopy
+gnu/usr.bin/binutils/objdump
+gnu/usr.bin/cc/c++
+gnu/usr.bin/cc/cc
+gnu/usr.bin/cc/cc1
+gnu/usr.bin/cc/cc1plus
+gnu/usr.bin/cc/cc_tools
+gnu/usr.bin/cc/cpp
+gnu/usr.bin/cc/gcov
+gnu/usr.bin/dialog
+gnu/usr.bin/diff3
+gnu/usr.bin/dtc
+gnu/usr.bin/gdb/gdb
+gnu/usr.bin/gdb/gdbserver
+gnu/usr.bin/gdb/gdbtui
+gnu/usr.bin/gdb/kgdb
+gnu/usr.bin/gperf
+gnu/usr.bin/grep
+kerberos5
+kerberos5/libexec/digest-service
+kerberos5/libexec/hprop
+kerberos5/libexec/hpropd
+kerberos5/libexec/ipropd-master
+kerberos5/libexec/ipropd-slave
+kerberos5/libexec/kadmind
+kerberos5/libexec/kcm
+kerberos5/libexec/kdc
+kerberos5/libexec/kdigest
+kerberos5/libexec/kfd
+kerberos5/libexec/kimpersonate
+kerberos5/libexec/kpasswdd
+kerberos5/tools/asn1_compile
+kerberos5/tools/make-roken
+kerberos5/tools/slc
+kerberos5/usr.bin/hxtool
+kerberos5/usr.bin/kadmin
+kerberos5/usr.bin/kcc
+kerberos5/usr.bin/kdestroy
+kerberos5/usr.bin/kf
+kerberos5/usr.bin/kgetcred
+kerberos5/usr.bin/kinit
+kerberos5/usr.bin/kpasswd
+kerberos5/usr.bin/ksu
+kerberos5/usr.bin/string2key
+kerberos5/usr.bin/verify_krb5_conf
+kerberos5/usr.sbin/iprop-log
+kerberos5/usr.sbin/kstash
+kerberos5/usr.sbin/ktutil
+lib/atf/libatf-c++/tests/detail
+lib/atf/libatf-c/tests/detail
+lib/libarchive/tests
+lib/libblacklist
+lib/libc/db/test
+lib/libc/tests/db
+lib/libc/tests/gen
+lib/libc/tests/gen/posix_spawn
+lib/libc/tests/hash
+lib/libc/tests/net
+lib/libc/tests/net/getaddrinfo
+lib/libc/tests/regex
+lib/libc/tests/ssp
+lib/libc/tests/stdio
+lib/libc/tests/stdlib
+lib/libproc/tests
+lib/libthr/tests
+lib/libxo/tests
+lib/msun/tests
+lib/ncurses/ncurses
+libexec/atf/atf-check
+libexec/atf/atf-sh
+libexec/atrun
+libexec/bootpd
+libexec/bootpd/bootpgw
+libexec/bootpd/tools/bootpef
+libexec/bootpd/tools/bootptest
+libexec/comsat
+libexec/dma/dma-mbox-create
+libexec/dma/dmagent
+libexec/fingerd
+libexec/ftpd
+libexec/getty
+libexec/mail.local
+libexec/mknetid
+libexec/pppoed
+libexec/rbootd
+libexec/revnetgroup
+libexec/rlogind
+libexec/rpc.rquotad
+libexec/rpc.rstatd
+libexec/rpc.rusersd
+libexec/rpc.rwalld
+libexec/rpc.sprayd
+libexec/rshd
+libexec/rtld-elf
+libexec/rtld-elf/tests/target
+libexec/smrsh
+libexec/talkd
+libexec/tcpd
+libexec/telnetd
+libexec/tftp-proxy
+libexec/tftpd
+libexec/ulog-helper
+libexec/ypxfr
+release/picobsd/tinyware/aps
+release/picobsd/tinyware/help
+release/picobsd/tinyware/login
+release/picobsd/tinyware/msg
+release/picobsd/tinyware/msh
+release/picobsd/tinyware/ns
+release/picobsd/tinyware/oinit
+release/picobsd/tinyware/passwd
+release/picobsd/tinyware/simple_httpd
+release/picobsd/tinyware/sps
+release/picobsd/tinyware/view
+release/picobsd/tinyware/vm
+rescue/rescue
+sbin/adjkerntz
+sbin/badsect
+sbin/bsdlabel
+sbin/camcontrol
+sbin/ccdconfig
+sbin/clri
+sbin/comcontrol
+sbin/conscontrol
+sbin/ddb
+sbin/decryptcore
+sbin/devd
+sbin/devfs
+sbin/dhclient
+sbin/dmesg
+sbin/dump
+sbin/dumpfs
+sbin/dumpon
+sbin/fdisk
+sbin/etherswitchcfg
+sbin/ffsinfo
+sbin/fsck
+sbin/fsck_ffs
+sbin/fsck_msdosfs
+sbin/fsdb
+sbin/fsirand
+sbin/gbde
+sbin/geom
+sbin/geom/core
+sbin/ggate/ggatec
+sbin/ggate/ggated
+sbin/ggate/ggatel
+sbin/growfs
+sbin/gvinum
+sbin/hastctl
+sbin/hastd
+sbin/ifconfig
+sbin/init
+sbin/ipf/ipf
+sbin/ipf/ipfs
+sbin/ipf/ipfstat
+sbin/ipf/ipftest
+sbin/ipf/ipmon
+sbin/ipf/ipnat
+sbin/ipf/ippool
+sbin/ipf/ipresend
+sbin/ipfw
+sbin/iscontrol
+sbin/kldconfig
+sbin/kldload
+sbin/kldstat
+sbin/kldunload
+sbin/ldconfig
+sbin/md5
+sbin/mdconfig
+sbin/mdmfs
+sbin/mknod
+sbin/mksnap_ffs
+sbin/mount
+sbin/mount_cd9660
+sbin/mount_fusefs
+sbin/mount_msdosfs
+sbin/mount_nfs
+sbin/mount_nullfs
+sbin/mount_udf
+sbin/mount_unionfs
+sbin/nandfs
+sbin/natd
+sbin/newfs
+sbin/newfs_msdos
+sbin/newfs_nandfs
+sbin/nfsiod
+sbin/nos-tun
+sbin/nvmecontrol
+sbin/pfctl
+sbin/pflogd
+sbin/ping
+sbin/ping6
+sbin/quotacheck
+sbin/rcorder
+sbin/reboot
+sbin/recoverdisk
+sbin/restore
+sbin/route
+sbin/routed
+sbin/routed/rtquery
+sbin/rtsol
+sbin/savecore
+sbin/sconfig
+sbin/setkey
+sbin/shutdown
+sbin/spppcontrol
+sbin/sunlabel
+sbin/swapon
+sbin/sysctl
+sbin/tunefs
+sbin/umount
+sbin/zfsbootcfg
+secure
+secure/libexec/sftp-server
+secure/libexec/ssh-keysign
+secure/libexec/ssh-pkcs11-helper
+secure/usr.bin/openssl
+secure/usr.bin/scp
+secure/usr.bin/sftp
+secure/usr.bin/ssh-add
+secure/usr.bin/ssh-agent
+secure/usr.bin/ssh-keygen
+secure/usr.bin/ssh-keyscan
+secure/usr.bin/ssh
+secure/usr.sbin/sshd
+share/examples/FreeBSD_version
+share/examples/find_interface
+share/examples/hwpmc
+share/examples/kld/cdev/test
+share/examples/kld/syscall/test
+share/examples/libvgl
+share/examples/perfmon
+share/examples/ppi
+share/examples/scsi_target
+share/examples/ses/getencstat
+share/examples/ses/sesd
+share/examples/ses/setencstat
+share/examples/ses/setobjstat
+sys/boot/efi/boot1
+sys/boot/efi/loader
+sys/boot/ficl
+sys/boot/i386/boot0
+sys/boot/i386/boot0sio
+sys/boot/i386/btx/btx
+sys/boot/i386/btx/btxldr
+sys/boot/i386/btx/lib
+sys/boot/i386/cdboot
+sys/boot/i386/gptboot
+sys/boot/i386/gptzfsboot
+sys/boot/i386/kgzldr
+sys/boot/i386/loader
+sys/boot/i386/mbr
+sys/boot/i386/pmbr
+sys/boot/i386/pxeldr
+sys/boot/i386/zfsboot
+sys/boot/mips/beri/loader
+sys/boot/powerpc/boot1.chrp
+sys/boot/powerpc/kboot
+sys/boot/powerpc/ofw
+sys/boot/powerpc/ps3
+sys/boot/powerpc/uboot
+sys/boot/sparc64/boot1
+sys/boot/sparc64/loader
+sys/boot/sparc64/zfsboot
+sys/boot/sparc64/zfsloader
+sys/boot/usb
+sys/boot/usb/tools
+sys/boot/userboot/ficl
+sys/boot/userboot/test
+sys/boot/zfs
+sys/contrib/xz-embedded/userspace
+sys/crypto/rijndael
+sys/dev/aic7xxx/aicasm
+sys/modules/aic7xxx/aicasm
+sys/teken/demo
+sys/teken/stress
+tests/sys/acl
+tests/sys/file
+tests/sys/fs/tmpfs
+tests/sys/kern
+tests/sys/kern/execve
+tests/sys/kqueue/libkqueue
+tests/sys/mqueue
+tests/sys/netinet
+tests/sys/pjdfstest/pjdfstest
+tools/KSE/ksetest
+tools/KSE/rr
+tools/bsdbox
+tools/diag/dumpvfscache
+tools/diag/localeck
+tools/regression/aio/aiop
+tools/regression/audit/audit_pipe_ioctl
+tools/regression/doat
+tools/regression/environ
+tools/regression/ethernet/ethermulti
+tools/regression/fsx
+tools/regression/gaithrstress
+tools/regression/geom/ConfCmp
+tools/regression/geom/MdLoad
+tools/regression/geom_gpt
+tools/regression/include/stdatomic
+tools/regression/include/tgmath
+tools/regression/kgssapi
+tools/regression/mlock
+tools/regression/netinet/arphold
+tools/regression/netinet/ipbroadcast
+tools/regression/netinet/ipdivert
+tools/regression/netinet/ipmulticast
+tools/regression/netinet/ipsockopt
+tools/regression/netinet/msocket
+tools/regression/netinet/msocket_ifnet_remove
+tools/regression/netinet/rawconnect
+tools/regression/netinet/tcpconnect
+tools/regression/netinet/tcpdrop
+tools/regression/netinet/tcpfullwindowrst
+tools/regression/netinet/tcpsockclosebeforeaccept
+tools/regression/netinet/tcpsocktimewait
+tools/regression/netinet/tcpstream
+tools/regression/netinet/udpconnectjail
+tools/regression/netinet/udpzerobyte
+tools/regression/netinet6/icmp6_filter
+tools/regression/netinet6/inet6_rth
+tools/regression/netinet6/ip6_sockets
+tools/regression/nfsmmap/test1
+tools/regression/nfsmmap/test2
+tools/regression/p1003_1b
+tools/regression/poll
+tools/regression/posixsem
+tools/regression/posixsem2
+tools/regression/priv
+tools/regression/pthread/cv_cancel1
+tools/regression/pthread/mutex_isowned_np
+tools/regression/rpcsec_gss
+tools/regression/security/access
+tools/regression/security/cap_test
+tools/regression/security/open_to_operation
+tools/regression/security/proc_to_proc
+tools/regression/sigqueue/sigqtest1
+tools/regression/sigqueue/sigqtest2
+tools/regression/sockets/accept_fd_leak
+tools/regression/sockets/accf_data_attach
+tools/regression/sockets/fstat
+tools/regression/sockets/kqueue
+tools/regression/sockets/listen_backlog
+tools/regression/sockets/listen_kqueue
+tools/regression/sockets/listenclose
+tools/regression/sockets/pr_atomic
+tools/regression/sockets/reconnect
+tools/regression/sockets/rtsocket
+tools/regression/sockets/sblock
+tools/regression/sockets/sendfile
+tools/regression/sockets/shutdown
+tools/regression/sockets/sigpipe
+tools/regression/sockets/so_setfib
+tools/regression/sockets/socketpair
+tools/regression/sockets/udp_pingpong
+tools/regression/sockets/unix_bindconnect
+tools/regression/sockets/unix_close_race
+tools/regression/sockets/unix_cmsg
+tools/regression/sockets/unix_gc
+tools/regression/sockets/unix_sendtorace
+tools/regression/sockets/unix_socket
+tools/regression/sockets/unix_sorflush
+tools/regression/sockets/zerosend
+tools/regression/sysvmsg
+tools/regression/sysvsem
+tools/regression/sysvshm
+tools/regression/tls/ttls1
+tools/regression/tls/ttls2
+tools/regression/tls/ttls4
+tools/regression/tmpfs
+tools/regression/ufs/uprintf
+tools/regression/usr.bin/env
+tools/test/auxinfo
+tools/test/iconv/gnu
+tools/test/iconv/posix
+tools/test/iconv/refgen
+tools/test/iconv/tablegen
+tools/test/malloc
+tools/test/net
+tools/test/netfibs
+tools/test/ppsapi
+tools/test/ptrace
+tools/test/upsdl
+tools/tools/aac
+tools/tools/ath/arcode
+tools/tools/ath/ath_ee_9287_print
+tools/tools/ath/ath_ee_9300_print
+tools/tools/ath/ath_ee_v14_print
+tools/tools/ath/ath_ee_v4k_print
+tools/tools/ath/ath_prom_read
+tools/tools/ath/athaggrstats
+tools/tools/ath/athalq
+tools/tools/ath/athdebug
+tools/tools/ath/athdecode
+tools/tools/ath/athkey
+tools/tools/ath/athpoke
+tools/tools/ath/athpow
+tools/tools/ath/athprom
+tools/tools/ath/athradar
+tools/tools/ath/athratestats
+tools/tools/ath/athrd
+tools/tools/ath/athregs
+tools/tools/ath/athspectral
+tools/tools/ath/athstats
+tools/tools/ath/athsurvey
+tools/tools/atsectl
+tools/tools/bootparttest
+tools/tools/bus_autoconf
+tools/tools/cfi
+tools/tools/cxgbtool
+tools/tools/decioctl
+tools/tools/dmardump
+tools/tools/drm/radeon/mkregtable
+tools/tools/ether_reflect
+tools/tools/find-sb
+tools/tools/fixwhite
+tools/tools/gdb_regofs
+tools/tools/gensnmpdef
+tools/tools/ifinfo
+tools/tools/ifpifa
+tools/tools/indent_wrapper
+tools/tools/ioat
+tools/tools/ipw
+tools/tools/iwi
+tools/tools/iwn/iwnstats
+tools/tools/kttcp
+tools/tools/mcgrab
+tools/tools/mctest
+tools/tools/mfi
+tools/tools/mwl/mwldebug
+tools/tools/mwl/mwlstats
+tools/tools/ncpus
+tools/tools/net80211/stumbler
+tools/tools/net80211/w00t/ap
+tools/tools/net80211/w00t/assoc
+tools/tools/net80211/w00t/expand
+tools/tools/net80211/w00t/prga
+tools/tools/net80211/w00t/redir
+tools/tools/net80211/wesside/dics
+tools/tools/net80211/wesside/udps
+tools/tools/net80211/wesside/wesside
+tools/tools/net80211/wlaninject
+tools/tools/net80211/wlanstats
+tools/tools/net80211/wlantxtime
+tools/tools/net80211/wlanwatch
+tools/tools/net80211/wlanwds
+tools/tools/netmap
+tools/tools/netrate/http
+tools/tools/netrate/httpd
+tools/tools/netrate/juggle
+tools/tools/netrate/netblast
+tools/tools/netrate/netreceive
+tools/tools/netrate/netsend
+tools/tools/netrate/tcpconnect
+tools/tools/netrate/tcpp
+tools/tools/netrate/tcpreceive
+tools/tools/npe/npestats
+tools/tools/nxge
+tools/tools/pciroms
+tools/tools/pirtool
+tools/tools/qrndtest
+tools/tools/syscall_timing
+tools/tools/tionxcl
+tools/tools/tscdrift
+tools/tools/umastat
+tools/tools/usbtest
+tools/tools/vimage
+tools/tools/vt/mkkfont
+tools/tools/vxge
+tools/tools/wtap/vis_map
+tools/tools/wtap/wtap
+tools/tools/zfsboottest
+tools/tools/smoketestsuite
+usr.bin/addr2line
+usr.bin/apply
+usr.bin/ar
+usr.bin/asa
+usr.bin/at
+usr.bin/atm/sscop
+usr.bin/awk
+usr.bin/banner
+usr.bin/basename
+usr.bin/bc
+usr.bin/biff
+usr.bin/bluetooth/bthost
+usr.bin/bluetooth/btsockstat
+usr.bin/bluetooth/rfcomm_sppd
+usr.bin/bmake
+usr.bin/brandelf
+usr.bin/bsdcat
+usr.bin/bsdcat/tests
+usr.bin/bsdiff/bsdiff
+usr.bin/bsdiff/bspatch
+usr.bin/bzip2
+usr.bin/bzip2recover
+usr.bin/c89
+usr.bin/c99
+usr.bin/caesar
+usr.bin/calendar
+usr.bin/cap_mkdb
+usr.bin/rlogin
+usr.bin/chat
+usr.bin/chkey
+usr.bin/chpass
+usr.bin/cksum
+usr.bin/clang/bugpoint
+usr.bin/clang/clang-format
+usr.bin/clang/clang-tblgen
+usr.bin/clang/clang
+usr.bin/clang/llc
+usr.bin/clang/lld
+usr.bin/clang/lldb
+usr.bin/clang/lli
+usr.bin/clang/llvm-ar
+usr.bin/clang/llvm-as
+usr.bin/clang/llvm-bcanalyzer
+usr.bin/clang/llvm-cov
+usr.bin/clang/llvm-cxxdump
+usr.bin/clang/llvm-cxxfilt
+usr.bin/clang/llvm-diff
+usr.bin/clang/llvm-dis
+usr.bin/clang/llvm-dwarfdump
+usr.bin/clang/llvm-extract
+usr.bin/clang/llvm-link
+usr.bin/clang/llvm-lto
+usr.bin/clang/llvm-lto2
+usr.bin/clang/opt
+usr.bin/clang/llvm-mc
+usr.bin/clang/llvm-modextract
+usr.bin/clang/llvm-nm
+usr.bin/clang/llvm-objdump
+usr.bin/clang/llvm-pdbutil
+usr.bin/clang/llvm-profdata
+usr.bin/clang/llvm-rtdyld
+usr.bin/clang/llvm-symbolizer
+usr.bin/clang/llvm-tblgen
+usr.bin/clang/llvm-xray
+usr.bin/cmp
+usr.bin/col
+usr.bin/colldef
+usr.bin/colrm
+usr.bin/column
+usr.bin/comm
+usr.bin/compile_et
+usr.bin/compress
+usr.bin/cpio
+usr.bin/cpio/tests
+usr.bin/cpuset
+usr.bin/csplit
+usr.bin/ctags
+usr.bin/ctlstat
+usr.bin/cut
+usr.bin/cxxfilt
+usr.bin/dc
+usr.bin/diff
+usr.bin/diff3
+usr.bin/dirname
+usr.bin/dpv
+usr.bin/drill
+usr.bin/dtc
+usr.bin/du
+usr.bin/ee
+usr.bin/elf2aout
+usr.bin/elfcopy
+usr.bin/elfdump
+usr.bin/enigma
+usr.bin/env
+usr.bin/expand
+usr.bin/factor
+usr.bin/false
+usr.bin/fetch
+usr.bin/file
+usr.bin/file2c
+usr.bin/find
+usr.bin/finger
+usr.bin/fmt
+usr.bin/fold
+usr.bin/fortune/fortune
+usr.bin/fortune/strfile
+usr.bin/fortune/unstr
+usr.bin/from
+usr.bin/fstat
+usr.bin/fsync
+usr.bin/ftp
+usr.bin/gcore
+usr.bin/gencat
+usr.bin/getaddrinfo
+usr.bin/getconf
+usr.bin/getconf/tests
+usr.bin/getent
+usr.bin/getopt
+usr.bin/gprof
+usr.bin/grdc
+usr.bin/grep
+usr.bin/gzip
+usr.bin/head
+usr.bin/hesinfo
+usr.bin/hexdump
+usr.bin/host
+usr.bin/iconv
+usr.bin/id
+usr.bin/ident
+usr.bin/indent
+usr.bin/ipcrm
+usr.bin/ipcs
+usr.bin/iscsictl
+usr.bin/join
+usr.bin/jot
+usr.bin/kdump
+usr.bin/keylogin
+usr.bin/keylogout
+usr.bin/killall
+usr.bin/ktrace
+usr.bin/ktrdump
+usr.bin/lam
+usr.bin/last
+usr.bin/lastcomm
+usr.bin/ldd
+usr.bin/leave
+usr.bin/less
+usr.bin/lessecho
+usr.bin/lesskey
+usr.bin/lex
+usr.bin/limits
+usr.bin/locale
+usr.bin/locate/bigram
+usr.bin/locate/code
+usr.bin/locate/locate
+usr.bin/localedef
+usr.bin/lock
+usr.bin/lockf
+usr.bin/logger
+usr.bin/login
+usr.bin/logins
+usr.bin/logname
+usr.bin/look
+usr.bin/lsvfs
+usr.bin/lzmainfo
+usr.bin/m4
+usr.bin/mail
+usr.bin/mandoc
+usr.bin/mesg
+usr.bin/minigzip
+usr.bin/ministat
+usr.bin/mkcsmapper
+usr.bin/mkcsmapper_static
+usr.bin/mkesdb
+usr.bin/mkesdb_static
+usr.bin/mkfifo
+usr.bin/mkimg
+usr.bin/mklocale
+usr.bin/mkstr
+usr.bin/mktemp
+usr.bin/mkuzip
+usr.bin/mt
+usr.bin/morse
+usr.bin/msgs
+usr.bin/nc
+usr.bin/ncal
+usr.bin/netstat
+usr.bin/newgrp
+usr.bin/newkey
+usr.bin/nfsstat
+usr.bin/nice
+usr.bin/nl
+usr.bin/nm
+usr.bin/nohup
+usr.bin/numactl
+usr.bin/number
+usr.bin/opieinfo
+usr.bin/opiekey
+usr.bin/opiepasswd
+usr.bin/pamtest
+usr.bin/passwd
+usr.bin/paste
+usr.bin/patch
+usr.bin/pathchk
+usr.bin/perror
+usr.bin/pom
+usr.bin/pr
+usr.bin/primes
+usr.bin/printenv
+usr.bin/printf
+usr.bin/proccontrol
+usr.bin/procstat
+usr.bin/procstat/tests
+usr.bin/rev
+usr.bin/protect
+usr.bin/quota
+usr.bin/random
+usr.bin/rctl
+usr.bin/readelf
+usr.bin/renice
+usr.bin/resizewin
+usr.bin/revoke
+usr.bin/rsh
+usr.bin/rpcgen
+usr.bin/rpcinfo
+usr.bin/rs
+usr.bin/xlint/lint1
+usr.bin/xlint/lint2
+usr.bin/xlint/xlint
+usr.bin/rup
+usr.bin/ruptime
+usr.bin/rusers
+usr.bin/rwall
+usr.bin/rwho
+usr.bin/script
+usr.bin/sdiff
+usr.bin/sed
+usr.bin/seq
+usr.bin/setchannel
+usr.bin/showmount
+usr.bin/size
+usr.bin/smbutil
+usr.bin/sockstat
+usr.bin/soelim
+usr.bin/sort
+usr.bin/split
+usr.bin/stat
+usr.bin/stdbuf
+usr.bin/strings
+usr.bin/su
+usr.bin/svn/svn
+usr.bin/svn/svnadmin
+usr.bin/svn/svnbench
+usr.bin/svn/svndumpfilter
+usr.bin/svn/svnfsfs
+usr.bin/svn/svnlook
+usr.bin/svn/svnmucc
+usr.bin/svn/svnrdump
+usr.bin/svn/svnserve
+usr.bin/svn/svnsync
+usr.bin/svn/svnversion
+usr.bin/systat
+usr.bin/tabs
+usr.bin/tail
+usr.bin/talk
+usr.bin/tar
+usr.bin/tar/tests
+usr.bin/tcopy
+usr.bin/tee
+usr.bin/telnet
+usr.bin/tftp
+usr.bin/time
+usr.bin/timeout
+usr.bin/tip/tip
+usr.bin/top
+usr.bin/touch
+usr.bin/tput
+usr.bin/tr
+usr.bin/true
+usr.bin/truncate
+usr.bin/truss
+usr.bin/tset
+usr.bin/tsort
+usr.bin/tty
+usr.bin/ul
+usr.bin/uname
+usr.bin/unexpand
+usr.bin/unifdef
+usr.bin/uniq
+usr.bin/units
+usr.bin/unvis
+usr.bin/unzip
+usr.bin/usbhidaction
+usr.bin/usbhidctl
+usr.bin/users
+usr.bin/uudecode
+usr.bin/uuencode
+usr.bin/vacation
+usr.bin/vgrind
+usr.bin/vi
+usr.bin/vis
+usr.bin/vmstat
+usr.bin/vtfontcvt
+usr.bin/w
+usr.bin/wall
+usr.bin/wc
+usr.bin/what
+usr.bin/whereis
+usr.bin/which
+usr.bin/who
+usr.bin/whois
+usr.bin/write
+usr.bin/xargs
+usr.bin/xinstall
+usr.bin/catman
+usr.bin/xo
+usr.bin/xstr
+usr.bin/xz
+usr.bin/xzdec
+usr.bin/yes
+usr.bin/yacc
+usr.bin/ypcat
+usr.bin/ypmatch
+usr.bin/ypwhich
+usr.bin/zstd
+usr.bin/xopo
+usr.bin/sdiotool
+usr.sbin/ac
+usr.sbin/accton
+usr.sbin/acpi/acpiconf
+usr.sbin/acpi/acpidb
+usr.sbin/acpi/acpidump
+usr.sbin/acpi/iasl
+usr.sbin/amd/amd
+usr.sbin/amd/amq
+usr.sbin/amd/fixmount
+usr.sbin/amd/fsinfo
+usr.sbin/amd/hlfsd
+usr.sbin/amd/mk-amd-map
+usr.sbin/amd/pawd
+usr.sbin/amd/wire-test
+usr.sbin/ancontrol
+usr.sbin/apm
+usr.sbin/apmd
+usr.sbin/arp
+usr.sbin/asf
+usr.sbin/audit
+usr.sbin/auditd
+usr.sbin/auditdistd
+usr.sbin/auditreduce
+usr.sbin/authpf
+usr.sbin/autofs
+usr.sbin/bhyve
+usr.sbin/bhyvectl
+usr.sbin/bhyveload
+usr.sbin/binmiscctl
+usr.sbin/blacklistctl
+usr.sbin/blacklistd
+usr.sbin/bluetooth/ath3kfw
+usr.sbin/bluetooth/bcmfw
+usr.sbin/bluetooth/bt3cfw
+usr.sbin/bluetooth/bthidcontrol
+usr.sbin/bluetooth/bthidd
+usr.sbin/bluetooth/btpand
+usr.sbin/bluetooth/hccontrol
+usr.sbin/bluetooth/hcsecd
+usr.sbin/bluetooth/hcseriald
+usr.sbin/bluetooth/l2control
+usr.sbin/bluetooth/l2ping
+usr.sbin/bluetooth/rfcomm_pppd
+usr.sbin/bluetooth/sdpcontrol
+usr.sbin/bluetooth/sdpd
+usr.sbin/boot0cfg
+usr.sbin/bootparamd/bootparamd
+usr.sbin/bootparamd/callbootd
+usr.sbin/bsdinstall/distextract
+usr.sbin/bsdinstall/distfetch
+usr.sbin/bsdinstall/partedit
+usr.sbin/bsnmpd/bsnmpd
+usr.sbin/bsnmpd/gensnmptree
+usr.sbin/bsnmpd/tools/bsnmptools
+usr.sbin/btxld
+usr.sbin/camdd
+usr.sbin/cdcontrol
+usr.sbin/chkgrp
+usr.sbin/chown
+usr.sbin/chroot
+usr.sbin/ckdist
+usr.sbin/clear_locks
+usr.sbin/config
+usr.sbin/cpucontrol
+usr.sbin/cron/cron
+usr.sbin/cron/crontab
+usr.sbin/crunch/crunchgen
+usr.sbin/crunch/crunchide
+usr.sbin/crunch/examples
+usr.sbin/ctladm
+usr.sbin/ctld
+usr.sbin/ctm/ctm
+usr.sbin/ctm/ctm_dequeue
+usr.sbin/ctm/ctm_rmail
+usr.sbin/ctm/ctm_smail
+usr.sbin/ctm/mkCTM
+usr.sbin/cxgbetool
+usr.sbin/daemon
+usr.sbin/dconschat
+usr.sbin/devctl
+usr.sbin/devinfo
+usr.sbin/digictl
+usr.sbin/diskinfo
+usr.sbin/dumpcis
+usr.sbin/editmap
+usr.sbin/edquota
+usr.sbin/eeprom
+usr.sbin/efidp
+usr.sbin/efivar
+usr.sbin/extattr
+usr.sbin/extattrctl
+usr.sbin/fdcontrol
+usr.sbin/fdformat
+usr.sbin/fdread
+usr.sbin/fdwrite
+usr.sbin/fifolog/fifolog_create
+usr.sbin/fifolog/fifolog_reader
+usr.sbin/fifolog/fifolog_writer
+usr.sbin/flowctl
+usr.sbin/fmtree
+usr.sbin/fstyp
+usr.sbin/ftp-proxy
+usr.sbin/fwcontrol
+usr.sbin/getfmac
+usr.sbin/getpmac
+usr.sbin/gpioctl
+usr.sbin/gssd
+usr.sbin/gstat
+usr.sbin/hyperv/tools/kvp
+usr.sbin/hyperv/tools/vss
+usr.sbin/i2c
+usr.sbin/ifmcstat
+usr.sbin/inetd
+usr.sbin/iostat
+usr.sbin/iovctl
+usr.sbin/ip6addrctl
+usr.sbin/ipfwpcap
+usr.sbin/iscsid
+usr.sbin/jail
+usr.sbin/jexec
+usr.sbin/jls
+usr.sbin/kbdcontrol
+usr.sbin/kbdmap
+usr.sbin/keyserv
+usr.sbin/kgmon
+usr.sbin/kgzip
+usr.sbin/kldxref
+usr.sbin/lastlogin
+usr.sbin/lmcconfig
+usr.sbin/lpr/chkprintcap
+usr.sbin/lpr/filters.ru/koi2855
+usr.sbin/lpr/filters.ru/koi2alt
+usr.sbin/lpr/filters
+usr.sbin/lpr/lpc
+usr.sbin/lpr/lpd
+usr.sbin/lpr/lpq
+usr.sbin/lpr/lpr
+usr.sbin/lpr/lprm
+usr.sbin/lpr/lptest
+usr.sbin/lpr/pac
+usr.sbin/lptcontrol
+usr.sbin/mailstats
+usr.sbin/mailwrapper
+usr.sbin/makefs
+usr.sbin/makemap
+usr.sbin/memcontrol
+usr.sbin/mfiutil
+usr.sbin/mixer
+usr.sbin/mld6query
+usr.sbin/mlxcontrol
+usr.sbin/mount_smbfs
+usr.sbin/mountd
+usr.sbin/moused
+usr.sbin/mpsutil
+usr.sbin/mptable
+usr.sbin/mptutil
+usr.sbin/mtest
+usr.sbin/nandsim
+usr.sbin/nandtool
+usr.sbin/ndiscvt
+usr.sbin/ndp
+usr.sbin/newsyslog
+usr.sbin/nfscbd
+usr.sbin/nfsd
+usr.sbin/nfsdumpstate
+usr.sbin/nfsrevoke
+usr.sbin/nfsuserd
+usr.sbin/ngctl
+usr.sbin/nghook
+usr.sbin/nmtree
+usr.sbin/nologin
+usr.sbin/nscd
+usr.sbin/ntp/ntp-keygen
+usr.sbin/ntp/ntpd
+usr.sbin/ntp/ntpdate
+usr.sbin/ntp/ntpdc
+usr.sbin/ntp/ntpq
+usr.sbin/ntp/ntptime
+usr.sbin/ntp/sntp
+usr.sbin/nvram
+usr.sbin/ofwdump
+usr.sbin/pciconf
+usr.sbin/pkg
+usr.sbin/pmcannotate
+usr.sbin/pmccontrol
+usr.sbin/pmcstat
+usr.sbin/pmcstudy
+usr.sbin/pnpinfo
+usr.sbin/portsnap/make_index
+usr.sbin/portsnap/phttpget
+usr.sbin/powerd
+usr.sbin/ppp
+usr.sbin/pppctl
+usr.sbin/praliases
+usr.sbin/praudit
+usr.sbin/procctl
+usr.sbin/prometheus_sysctl_exporter
+usr.sbin/pstat
+usr.sbin/pw
+usr.sbin/pw/tests
+usr.sbin/pwd_mkdb
+usr.sbin/quot
+usr.sbin/quotaon
+usr.sbin/rarpd
+usr.sbin/repquota
+usr.sbin/rip6query
+usr.sbin/rmt
+usr.sbin/route6d
+usr.sbin/rpc.lockd
+usr.sbin/sa
+usr.sbin/rpc.statd
+usr.sbin/rpc.umntall
+usr.sbin/rpc.yppasswdd
+usr.sbin/rpc.ypupdated
+usr.sbin/rpc.ypxfrd
+usr.sbin/rpcbind
+usr.sbin/rrenumd
+usr.sbin/rtadvctl
+usr.sbin/rtadvd
+usr.sbin/rtprio
+usr.sbin/rtsold
+usr.sbin/rwhod
+usr.sbin/sendmail
+usr.sbin/services_mkdb
+usr.sbin/sesutil
+usr.sbin/setfib
+usr.sbin/setfmac
+usr.sbin/setpmac
+usr.sbin/smbmsg
+usr.sbin/snapinfo
+usr.sbin/spray
+usr.sbin/syslogd
+usr.sbin/tcpdchk
+usr.sbin/tcpdmatch
+usr.sbin/tcpdrop
+usr.sbin/tcpdump/tcpdump
+usr.sbin/timed/timed
+usr.sbin/timed/timedc
+usr.sbin/traceroute
+usr.sbin/traceroute6
+usr.sbin/trpt
+usr.sbin/tzsetup
+usr.sbin/uathload
+usr.sbin/uefisign
+usr.sbin/ugidfw
+usr.sbin/uhsoctl
+usr.sbin/unbound/anchor
+usr.sbin/unbound/checkconf
+usr.sbin/unbound/control
+usr.sbin/unbound/daemon
+usr.sbin/usbconfig
+usr.sbin/usbdump
+usr.sbin/utx
+usr.sbin/vidcontrol
+usr.sbin/vipw
+usr.sbin/wake
+usr.sbin/watch
+usr.sbin/watchdogd
+usr.sbin/wlandebug
+usr.sbin/wpa/hostapd
+usr.sbin/wpa/hostapd_cli
+usr.sbin/wpa/ndis_events
+usr.sbin/wpa/wpa_cli
+usr.sbin/wpa/wpa_passphrase
+usr.sbin/wpa/wpa_priv
+usr.sbin/wpa/wpa_supplicant
+usr.sbin/yp_mkdb
+usr.sbin/ypbind
+usr.sbin/ypldap
+usr.sbin/yppoll
+usr.sbin/yppush
+usr.sbin/ypserv
+usr.sbin/ypset
+usr.sbin/zic/zdump
+usr.sbin/zic/zic
+usr.sbin/zonectl
Index: tools/tools/smoketestsuite/utils.h
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/utils.h
@@ -0,0 +1,83 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _UTILS_H_
+#define _UTILS_H_
+
+#include <unordered_map>
+#include <vector>
+
+namespace utils {
+ /*
+ * Option relation which maps option names to
+ * a unique identifier in their description.
+ */
+ struct OptRelation {
+ char type; /* Option type: (s)short/(l)long. */
+ std::string value; /* Name of the option. */
+ /* The keyword which should be looked up in usage
+ * message (if) produced when using this options.
+ */
+ std::string keyword;
+ };
+
+ /*
+ * Read/Write file descriptors for a pipe.
+ */
+ struct PipeDescriptor {
+ int readfd;
+ int writefd;
+ pid_t pid; /* PID of the forked shell process. */
+ };
+
+ /*
+ * Temporary directory inside which the utility-specific
+ * commands will be executed, and all the side effects
+ * (core dumps, executables etc.) that are created will
+ * be restricted in this directory.
+ */
+ extern const char *tmpdir;
+
+ std::string GenerateCommand(std::string, std::string);
+ std::pair<std::string, int> Execute(std::string);
+ PipeDescriptor* POpen(const char*);
+
+ class OptDefinition {
+ public:
+ /* List of all the accepted options with unknown usage. */
+ std::vector<std::string> opt_list;
+ /* Map "option value" to "option definition". */
+ std::unordered_map<std::string, OptRelation> opt_map;
+ std::unordered_map<std::string, OptRelation>::iterator opt_map_iter;
+
+ void InsertOpts();
+ std::vector<OptRelation *> CheckOpts(std::string);
+ };
+}
+
+#endif /* _UTILS_H_ */
Index: tools/tools/smoketestsuite/utils.cpp
===================================================================
--- /dev/null
+++ tools/tools/smoketestsuite/utils.cpp
@@ -0,0 +1,336 @@
+/*-
+ * Copyright 2017-2018 Shivansh Rai
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <fcntl.h>
+#include <signal.h>
+#include <string.h>
+#include <sys/select.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <array>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+
+#include "utils.h"
+#include "fetch_groff.h"
+#include "logging.h"
+
+#define READ 0 /* Pipe descriptor: read end. */
+#define WRITE 1 /* Pipe descriptor: write end. */
+/*
+ * Buffer size (used for buffering output generated
+ * after executing the utility-specific command).
+ */
+#define BUFSIZE 128
+#define TIMEOUT 1 /* Threshold (seconds) for a function call to return. */
+
+const char *utils::tmpdir = "tmpdir";
+/*
+ * Insert a list of user-defined option definitions
+ * into a hashmap. These specific option definitions
+ * are the ones which one can be easily tested.
+ */
+void
+utils::OptDefinition::InsertOpts()
+{
+ /* Option definitions. */
+ OptRelation h_def; /* '-h' */
+ h_def.type = 's';
+ h_def.value = "h";
+ h_def.keyword = "help";
+
+ OptRelation v_def; /* '-v' */
+ v_def.type = 's';
+ v_def.value = "v";
+ v_def.keyword = "version";
+
+ /*
+ * "opt_map" contains all the options
+ * which can be easily tested.
+ */
+ opt_map.insert(std::make_pair<std::string, OptRelation>
+ ("h", (OptRelation)h_def));
+ opt_map.insert(std::make_pair<std::string, OptRelation>
+ ("v", (OptRelation)v_def));
+};
+
+/*
+ * For the utility under test, find the supported options
+ * present in the hashmap generated by InsertOpts()
+ * and return them in a form of list of option relations.
+ */
+std::vector<utils::OptRelation *>
+utils::OptDefinition::CheckOpts(std::string utility)
+{
+ std::string opt_id = ".It Fl"; /* Option identifier in man page. */
+ std::string line; /* An individual line in a man-page. */
+ std::string opt_name; /* Name of the option. */
+ std::string buffer; /* Option description extracted from man-page. */
+ std::string opt_string; /* Identified option names. */
+ int opt_pos; /* Starting index of the (identified) option. */
+ int space_index; /* First occurrence of space in option definition. */
+ std::vector<OptRelation *> identified_opts;
+ std::vector<std::string> supported_sections = { "1", "8" };
+
+ /* Generate the hashmap "opt_map". */
+ InsertOpts();
+ std::ifstream infile(groff::groff_map[utility]);
+
+ /*
+ * Search for all the options accepted by the
+ * utility and collect those present in "opt_map".
+ */
+ while (std::getline(infile, line)) {
+ if ((opt_pos = line.find(opt_id)) != std::string::npos) {
+ /* Locate the position of option name. */
+ opt_pos += opt_id.size() + 1;
+ if (opt_pos > line.size()) {
+ /*
+ * This condition will trigger when a utility
+ * supports an empty argument, e.g. tset(issue #9)
+ */
+ continue;
+ }
+
+ /*
+ * Check for long options ; While here, also sanitize
+ * multi-word option definitions in a man page to properly
+ * extract short options from option definitions such as:
+ * .It Fl r Ar seconds (taken from date(1)).
+ */
+ if ((space_index = line.find(" ", opt_pos + 1, 1))
+ != std::string::npos)
+ opt_name = line.substr(opt_pos, space_index - opt_pos);
+ else
+ opt_name = line.substr(opt_pos);
+
+ /*
+ * Check if the identified option matches the identifier.
+ * "opt_list.back()" is the previously checked option, the
+ * description of which is now stored in "buffer".
+ */
+ if (!opt_list.empty() &&
+ (opt_map_iter = opt_map.find(opt_list.back()))
+ != opt_map.end() &&
+ buffer.find((opt_map_iter->second).keyword) != std::string::npos) {
+ identified_opts.push_back(&(opt_map_iter->second));
+ /*
+ * Since the usage of the option under test
+ * is known, we remove it from "opt_list".
+ */
+ opt_list.pop_back();
+ }
+
+ /* Update the list of valid options. */
+ opt_list.push_back(opt_name);
+ /* Empty the buffer for next option's description. */
+ buffer.clear();
+ } else {
+ /*
+ * Collect the option description until next
+ * valid option definition is encountered.
+ */
+ buffer.append(line);
+ }
+ }
+
+ return identified_opts;
+}
+
+/* Generates command for execution. */
+std::string
+utils::GenerateCommand(std::string utility, std::string opt)
+{
+ std::string command = utility;
+
+ if (!opt.empty())
+ command += " -" + opt;
+ command += " 2>&1 </dev/null";
+
+ return command;
+}
+
+/*
+ * When pclose() is called on the stream returned by popen(),
+ * it waits indefinitely for the created shell process to
+ * terminate in cases where the shell command waits for the
+ * user input via a blocking read (e.g. passwd(1)).
+ * Hence, we define a custom function which alongside
+ * returning the read-write file descriptors, also returns
+ * the pid of the newly created (child) shell process. This
+ * pid can later be used for sending a signal to the child.
+ *
+ * NOTE For the sake of completeness, we also specify actions
+ * to be taken corresponding to the "w" (write) type. However,
+ * in this context only the "r" (read) type will be required.
+ */
+
+utils::PipeDescriptor*
+utils::POpen(const char *command)
+{
+ int pdes[2];
+ char *argv[4];
+ pid_t child_pid;
+ PipeDescriptor *pipe_descr = (PipeDescriptor *)malloc(sizeof(PipeDescriptor));
+
+ /* Create a pipe with ~
+ * - pdes[READ]: read end
+ * - pdes[WRITE]: write end
+ */
+ if (pipe2(pdes, O_CLOEXEC) < 0)
+ return NULL;
+
+ pipe_descr->readfd = pdes[READ];
+ pipe_descr->writefd = pdes[WRITE];
+
+ /* Type-cast to avoid compiler warnings [-Wwrite-strings]. */
+ argv[0] = (char *)"sh";
+ argv[1] = (char *)"-c";
+ argv[2] = (char *)command;
+ argv[3] = NULL;
+
+ switch (child_pid = vfork()) {
+ case -1: /* Error. */
+ free(pipe_descr);
+ close(pdes[READ]);
+ close(pdes[WRITE]);
+ return NULL;
+ case 0: /* Child. */
+ /*
+ * TODO Verify if the following operations on both read
+ * and write file-descriptors (irrespective of the value
+ * of 'type', which is no longer being used) is safe.
+ */
+ if (pdes[WRITE] != STDOUT_FILENO)
+ dup2(pdes[WRITE], STDOUT_FILENO);
+ else
+ fcntl(pdes[WRITE], F_SETFD, 0);
+ if (pdes[READ] != STDIN_FILENO)
+ dup2(pdes[READ], STDIN_FILENO);
+ else
+ fcntl(pdes[READ], F_SETFD, 0);
+
+ /*
+ * For current usecase, it might so happen that the child gets
+ * stuck on a blocking read (e.g. passwd(1)) waiting for user
+ * input. In that case the child will be killed via a signal.
+ * To avoid any effect on the parent's execution, we place the
+ * child in a separate process group with pgid set as "child_pid".
+ */
+ setpgid(child_pid, child_pid);
+ execve("/bin/sh", argv, NULL);
+ exit(127);
+ }
+
+ pipe_descr->pid = child_pid;
+ return pipe_descr;
+}
+
+/*
+ * Executes the command passed as argument in a
+ * shell and returns its output and exit status.
+ */
+std::pair<std::string, int>
+utils::Execute(std::string command)
+{
+ int result;
+ int exitstatus;
+ std::array<char, BUFSIZE> buffer;
+ std::string usage_output;
+ struct timeval tv;
+ fd_set readfds;
+ FILE *pipe;
+ PipeDescriptor *pipe_descr;
+ pid_t pid;
+ int pstat;
+
+ /* Execute "command" inside "tmpdir". */
+ chdir(tmpdir);
+ pipe_descr = utils::POpen(command.c_str());
+ chdir("..");
+
+ if (pipe_descr == NULL) {
+ logging::LogPerror("utils::POpen()");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Close the unrequired file-descriptor. */
+ close(pipe_descr->writefd);
+
+ pipe = fdopen(pipe_descr->readfd, "r");
+ free(pipe_descr);
+ if (pipe == NULL) {
+ close(pipe_descr->readfd);
+ logging::LogPerror("fdopen()");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Set a timeout for shell process to complete its execution. */
+ tv.tv_sec = TIMEOUT;
+ tv.tv_usec = 0;
+
+ FD_ZERO(&readfds);
+ FD_SET(fileno(pipe), &readfds);
+ result = select(fileno(pipe) + 1, &readfds, NULL, NULL, &tv);
+
+ if (result > 0) {
+ while (!feof(pipe) && !ferror(pipe)) {
+ if (fgets(buffer.data(), BUFSIZE, pipe) != NULL)
+ usage_output += buffer.data();
+ }
+ } else if (result == -1) {
+ logging::LogPerror("select()");
+ if (kill(pipe_descr->pid, SIGTERM) < 0)
+ logging::LogPerror("kill()");
+ } else if (result == 0) {
+ /*
+ * We gave a relaxed value of TIMEOUT seconds for the shell process
+ * to complete it's execution. If at this point it is still
+ * alive, it (most probably) is stuck on a blocking read
+ * waiting for the user input. Since a few of the utilities
+ * performing such blocking reads don't respond to SIGINT
+ * (e.g. pax(1)), we terminate the shell process via SIGTERM.
+ */
+ if (kill(pipe_descr->pid, SIGTERM) < 0)
+ logging::LogPerror("kill()");
+ }
+
+ /* Retrieve exit status of the shell process. */
+ do {
+ pid = wait4(pipe_descr->pid, &pstat, 0, (struct rusage *)0);
+ } while (pid == -1 && errno == EINTR);
+
+ fclose(pipe);
+ exitstatus = (pid == -1) ? -1 : WEXITSTATUS(pstat);
+ DEBUGP("Command: %s, exit status: %d\n", command.c_str(), exitstatus);
+
+ return std::make_pair<std::string, int>
+ ((std::string)usage_output, (int)exitstatus);
+}

File Metadata

Mime Type
text/plain
Expires
Mon, Aug 17, 9:10 PM (6 h, 41 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
36872335
Default Alt Text
D12249.id37452.diff (90 KB)

Event Timeline