Index: head/devel/ccache/Makefile =================================================================== --- head/devel/ccache/Makefile (revision 473409) +++ head/devel/ccache/Makefile (revision 473410) @@ -1,126 +1,145 @@ # Created by: Dominic Marks # $FreeBSD$ PORTNAME= ccache -PORTVERSION= 3.3.5 -PORTREVISION= 2 +PORTVERSION= 3.4.2 +PORTREVISION= 0 CATEGORIES= devel MASTER_SITES= https://www.samba.org/ftp/ccache/ \ LOCAL/bdrewery MAINTAINER= bdrewery@FreeBSD.org COMMENT= Tool to minimize the compile time of C/C++ programs LICENSE= GPLv3+ LICENSE_FILE= ${WRKSRC}/GPL-3.0.txt -CONFLICTS_INSTALL= ccache-static-[0-9]* ccache-memcached-[0-9]* ccache-memcached-static-[0-9]* +CONFLICTS_INSTALL?= ccache-static-[0-9]* ccache-memcached-[0-9]* ccache-memcached-static-[0-9]* GNU_CONFIGURE= yes HOWTO= ccache-howto-freebsd.txt CCLINKDIR= libexec/ccache SUB_FILES= ${HOWTO} world-ccache pkg-message ccache-update-links.sh -PORTDOCS= ccache-howto-freebsd.txt MANUAL.html MANUAL.txt +PORTDOCS= ccache-howto-freebsd.txt MANUAL.html MANUAL.adoc NEWS.adoc \ + NEWS.html OPTIONS_DEFINE= CLANGLINK LLVMLINK STATIC DOCS TINDERBOX MEMCACHED OPTIONS_DEFAULT=CLANGLINK LLVMLINK CLANGLINK_DESC= Create clang compiler links if clang is installed LLVMLINK_DESC= Create llvm compiler links if llvm is installed TINDERBOX_DESC= Create tarball for tinderbox usage MEMCACHED_DESC= Build in experimental Memcached support USES= compiler -MEMCACHED_EXTRA_PATCHES= ${FILESDIR}/extra-patch-memcached:-p1 \ - ${FILESDIR}/extra-patch-memcached-ccache.c \ +MEMCACHED_EXTRA_PATCHES= \ ${FILESDIR}/extra-patch-memcached-configure.ac \ ${FILESDIR}/extra-patch-memcached-Makefile.in MEMCACHED_CONFIGURE_ENABLE= memcached -MEMCACHED_USES= autoreconf pkgconfig +MEMCACHED_USES= autoreconf gmake pkgconfig MEMCACHED_LIB_DEPENDS= libmemcached.so:databases/libmemcached MEMCACHED_LDFLAGS= -L${LOCALBASE}/lib MEMCACHED_CFLAGS= -I${LOCALBASE}/include STATIC_CONFIGURE_ENABLE= static .if defined(WITH_CCACHE_BUILD) && empty(OPTIONS_SLAVE:MMEMCACHED) && \ !defined(NO_CCACHE_DEPEND) # Don't allow autoreconf. We want no dependencies on this to keep # WITH_CCACHE_BUILD working. USES:= ${USES:Nautoreconf} MEMCACHED_IGNORE= MEMCACHED cannot be combined with WITH_CCACHE_BUILD. Use devel/ccache-memcached # XXX: This needs more testing with Poudriere before enabling. Also bsd.options.mk support. #MEMCACHED_DEPENDS_ARGS+= NO_CCACHE_DEPEND=1 .endif # Support WITH_CCACHE_BUILD but don't depend on it. NO_CCACHE_DEPEND= yes OPTIONS_SUB= yes STATIC_LDFLAGS= -static +.include + +.if ${PORT_OPTIONS:MMEMCACHED} +USE_GITHUB= yes +GH_TAGNAME= dd9123d3bc0763ef7133330e20ca3e3409a08904 +MASTER_SITES= +DISTINFO_FILE= ${MASTERDIR}/distinfo.memcached +BUILD_DEPENDS+= ${LOCALBASE}/bin/a2x:textproc/asciidoc +ALL_TARGET= all docs +.endif + .include . if ${COMPILER_TYPE} == clang CPPFLAGS+= -DCC_IS_CLANG . elif ${COMPILER_TYPE} == gcc CPPFLAGS+= -DCC_IS_GCC . endif PLIST_SUB+= CCLINKDIR="${CCLINKDIR}" .if ${ARCH}=="i386" CCACHE_COMPILERS+= icc icpc .endif GNU_COMPILERS+= 34 42 43 44 45 46 47 48 49 5 6 7 8 CCACHE_COMPILERS+= cc c++ CC gcc g++ ${GNU_COMPILERS:S|^|gcc|} ${GNU_COMPILERS:S|^|g++|} .if ${PORT_OPTIONS:MCLANGLINK} CLANG_COMPILERS+= 33 34 35 36 37 38 39 40 50 60 -devel CCACHE_COMPILERS+= clang clang++ ${CLANG_COMPILERS:S|^|clang|} ${CLANG_COMPILERS:S|^|clang++|} .endif .if ${PORT_OPTIONS:MLLVMLINK} CCACHE_COMPILERS+= llvm-gcc llvm-c++ llvm-g++ .endif CCACHE_COMPILERS+= ${EXTRA_COMPILERS} SUB_LIST+= CCACHE_COMPILERS="${CCACHE_COMPILERS}" \ CCLINKDIR="${CCLINKDIR}" \ ICCPREFIX="${LOCALBASE}/intel_cc_80/bin" \ HOWTO="${HOWTO}" +post-patch-MEMCACHED-off: + @${REINPLACE_CMD} \ + -e 's,_XOPEN_SOURCE 600,_XOPEN_SOURCE 700,' \ + -e 's,_POSIX_C_SOURCE 200112L,_POSIX_C_SOURCE 200809L,' \ + ${WRKSRC}/configure + post-build-TINDERBOX-on: @${MKDIR} ${WRKDIR}/tb/opt @${INSTALL_PROGRAM} ${WRKSRC}/${PORTNAME} ${WRKDIR}/tb/opt .for l in ${CCACHE_COMPILERS} @${LN} -sf ${PORTNAME} ${WRKDIR}/tb/opt/${l} .endfor @${TAR} -C ${WRKDIR}/tb -cpf ${WRKSRC}/${PORTNAME}.tar opt do-install: ${INSTALL_PROGRAM} ${WRKSRC}/ccache ${STAGEDIR}${PREFIX}/bin - ${INSTALL_MAN} ${WRKSRC}/ccache.1 ${STAGEDIR}${PREFIX}/man/man1 + ${INSTALL_MAN} ${WRKSRC}/doc/ccache.1 ${STAGEDIR}${PREFIX}/man/man1 ${MKDIR} ${STAGEDIR}${PREFIX}/${CCLINKDIR}/world ${INSTALL_SCRIPT} ${WRKDIR}/world-ccache \ ${STAGEDIR}${PREFIX}/${CCLINKDIR}/world/ccache ${INSTALL_SCRIPT} ${WRKDIR}/ccache-update-links.sh \ ${STAGEDIR}${PREFIX}/bin/ccache-update-links do-install-TINDERBOX-on: ${MKDIR} ${STAGEDIR}${DATADIR} ${INSTALL_DATA} ${WRKSRC}/${PORTNAME}.tar ${STAGEDIR}${DATADIR} do-install-DOCS-on: ${MKDIR} ${STAGEDIR}${DOCSDIR} - ${INSTALL_DATA} ${WRKSRC}/MANUAL.html ${STAGEDIR}${DOCSDIR} - ${INSTALL_DATA} ${WRKSRC}/MANUAL.txt ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${WRKSRC}/doc/MANUAL.html ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${WRKSRC}/doc/MANUAL.adoc ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${WRKSRC}/doc/NEWS.adoc ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${WRKSRC}/doc/NEWS.html ${STAGEDIR}${DOCSDIR} ${INSTALL_DATA} ${WRKDIR}/${HOWTO} ${STAGEDIR}${DOCSDIR} .include Index: head/devel/ccache/distinfo =================================================================== --- head/devel/ccache/distinfo (revision 473409) +++ head/devel/ccache/distinfo (revision 473410) @@ -1,3 +1,3 @@ -TIMESTAMP = 1516823791 -SHA256 (ccache-3.3.5.tar.gz) = c84b2460dab2bd9b8b743499a51910c6b7149bc0271d0f6000e0ad1f6c8fda2b -SIZE (ccache-3.3.5.tar.gz) = 453896 +TIMESTAMP = 1530039944 +SHA256 (ccache-3.4.2.tar.gz) = b2264923c63e2b90a17cf56acb1df3f4229c416fb88e476e5ec7e02919d319c3 +SIZE (ccache-3.4.2.tar.gz) = 460637 Index: head/devel/ccache/distinfo.memcached =================================================================== --- head/devel/ccache/distinfo.memcached (nonexistent) +++ head/devel/ccache/distinfo.memcached (revision 473410) @@ -0,0 +1,3 @@ +TIMESTAMP = 1530040182 +SHA256 (ccache-ccache-3.4.2-dd9123d3bc0763ef7133330e20ca3e3409a08904_GH0.tar.gz) = 842b55b091f21878c4d4eb3900fa7501f632af5d39d42b5cb81ce47d231da40f +SIZE (ccache-ccache-3.4.2-dd9123d3bc0763ef7133330e20ca3e3409a08904_GH0.tar.gz) = 358431 Property changes on: head/devel/ccache/distinfo.memcached ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +yes \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/devel/ccache/files/patch-configure =================================================================== --- head/devel/ccache/files/patch-configure (revision 473409) +++ head/devel/ccache/files/patch-configure (nonexistent) @@ -1,20 +0,0 @@ ---- configure.orig 2015-10-08 19:14:23 UTC -+++ configure -@@ -3672,7 +3672,7 @@ $as_echo "#define _XOPEN_SOURCE 500" >>c - ;; - *) - --$as_echo "#define _XOPEN_SOURCE 600" >>confdefs.h -+$as_echo "#define _XOPEN_SOURCE 700" >>confdefs.h - - ;; - esac -@@ -3694,7 +3694,7 @@ $as_echo "#define _XOPEN_SOURCE_EXTENDED - esac - - --$as_echo "#define _POSIX_C_SOURCE 200112L" >>confdefs.h -+$as_echo "#define _POSIX_C_SOURCE 200809L" >>confdefs.h - - - fi Property changes on: head/devel/ccache/files/patch-configure ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -yes \ No newline at end of property Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/devel/ccache/files/extra-patch-memcached-ccache.c =================================================================== --- head/devel/ccache/files/extra-patch-memcached-ccache.c (revision 473409) +++ head/devel/ccache/files/extra-patch-memcached-ccache.c (nonexistent) @@ -1,11 +0,0 @@ ---- ccache.c.orig 2017-03-09 09:08:38.803365000 -0800 -+++ ccache.c 2017-03-09 09:08:42.059821000 -0800 -@@ -2153,7 +2153,7 @@ calculate_object_hash(struct args *args, - conf->direct_mode = false; - return NULL; - } -- char *manifest_name = hash_result(hash); -+ manifest_name = hash_result(hash); - manifest_path = get_path_in_cache(manifest_name, ".manifest"); - /* Check if the manifest file is there. */ - struct stat st; Property changes on: head/devel/ccache/files/extra-patch-memcached-ccache.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -yes \ No newline at end of property Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/devel/ccache/files/patch-ccache.c =================================================================== --- head/devel/ccache/files/patch-ccache.c (revision 473409) +++ head/devel/ccache/files/patch-ccache.c (nonexistent) @@ -1,62 +0,0 @@ -- Determine whether cc(1) is clang or gcc at compile-time. -- Don't hash -fcolor-diagnostics; make will auto use it while make -j will not. - There's no reason to not use the cache in either of these cases if it is - already available. --bdrewery - ---- ccache.c.orig 2016-10-26 14:31:19.000000000 -0700 -+++ ccache.c 2017-02-03 14:24:35.466505000 -0800 -@@ -1468,6 +1468,11 @@ compiler_is_clang(struct args *args) - { - char *name = basename(args->argv[0]); - bool result = strstr(name, "clang") != NULL; -+#ifdef CC_IS_CLANG -+ if (strcmp(name, "cc") == 0 || strcmp(name, "CC") == 0 || -+ strcmp(name, "c++") == 0) -+ result = true; -+#endif - free(name); - return result; - } -@@ -1477,6 +1482,11 @@ compiler_is_gcc(struct args *args) - { - char *name = basename(args->argv[0]); - bool result = strstr(name, "gcc") || strstr(name, "g++"); -+#ifdef CC_IS_GCC -+ if (strcmp(name, "cc") == 0 || strcmp(name, "CC") == 0 || -+ strcmp(name, "c++") == 0) -+ result = true; -+#endif - free(name); - return result; - } -@@ -1592,6 +1602,7 @@ calculate_common_hash(struct args *args, - free(p); - } - -+#if 0 - // Possibly hash GCC_COLORS (for color diagnostics). - if (compiler_is_gcc(args)) { - const char *gcc_colors = getenv("GCC_COLORS"); -@@ -1600,6 +1611,7 @@ calculate_common_hash(struct args *args, - hash_string(hash, gcc_colors); - } - } -+#endif - } - - // Update a hash sum with information specific to the direct and preprocessor -@@ -1629,6 +1641,13 @@ calculate_object_hash(struct args *args, - continue; - } - -+ /* Colors do not affect compilation. */ -+ if (str_startswith(args->argv[i], "-fcolor-diagnostics") || -+ str_eq(args->argv[i], "-fdiagnostics-color") || -+ str_eq(args->argv[i], "-fdiagnostics-color=always")) { -+ continue; -+ } -+ - // The -fdebug-prefix-map option may be used in combination with - // CCACHE_BASEDIR to reuse results across different directories. Skip it - // from hashing. Property changes on: head/devel/ccache/files/patch-ccache.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -yes \ No newline at end of property Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/devel/ccache/files/extra-patch-memcached =================================================================== --- head/devel/ccache/files/extra-patch-memcached (revision 473409) +++ head/devel/ccache/files/extra-patch-memcached (nonexistent) @@ -1,2387 +0,0 @@ -https://github.com/ccache/ccache/pull/58 -Retrieved on February 13th 2017. -Changes to .travis.yml removed since it is not in the release image. - -diff --git a/MANUAL.txt b/MANUAL.txt -index ab01886..c78bb6e 100644 ---- a/MANUAL.txt -+++ b/MANUAL.txt -@@ -418,6 +418,20 @@ WRAPPERS>>. - The default value is 5G. Available suffixes: k, M, G, T (decimal) and Ki, - Mi, Gi, Ti (binary). The default suffix is "G". - -+*memcached_conf* (*CCACHE_MEMCACHED_CONF*):: -+ -+ The memcached_conf option sets the memcached(3) configuration to use for -+ storing and getting cache values, if any. Example configuration: -++ -+------------------------------------------------------------------------------- -+CCACHE_MEMCACHED_CONF=--SERVER=localhost:11211 -+------------------------------------------------------------------------------- -+ -+*memcached_only* (*CCACHE_MEMCACHED_ONLY*):: -+ -+ Only store files in memcached, don't store them in the local filesystems. -+ The manifests (for direct mode) and stats are still being stored locally. -+ - *path* (*CCACHE_PATH*):: - - If set, ccache will search directories in this list when looking for the -@@ -451,6 +465,11 @@ WRAPPERS>>. - from the cache using the direct mode, not the preprocessor mode. See - documentation for *read_only* regarding using a read-only ccache directory. - -+*read_only_memcached* (*CCACHE_READONLY_MEMCACHED* or *CCACHE_NOREADONLY_MEMCACHED*), see <<_boolean_values,Boolean values>> above):: -+ -+ If true, ccache will attempt to get previously cached values from memcached, -+ but will not try to store any new values in memcached. -+ - *recache* (*CCACHE_RECACHE* or *CCACHE_NORECACHE*, see <<_boolean_values,Boolean values>> above):: - - If true, ccache will not use any previously stored result. New results will -@@ -769,6 +788,29 @@ A tip is to set *temporary_dir* to a directory on the local host to avoid NFS - traffic for temporary files. - - -+Sharing a cache with memcached -+------------------------------ -+ -+When using the *memcached* () feature, the most recently -+used cache entries are also available from the configured memcached servers. -+ -+The local cache directory will be searched first, but then it will still be -+possible to get cache hits (over the network) before having to run the -+compiler. -+ -+Using a local *moxi* (memcached proxy) will enable multiple ccache invocations -+to share memcached connections and thus avoid some of the network overhead. -+ -+It will also allow you to fine-tune connection timeouts and other settings. You -+can optionally replace your memcached servers with Couchbase servers. -+ -+Example: -+ -+------------------------------------------------------------------------------- -+moxi -z 11211=mc_server1:11211,mc_server2:11211 -+------------------------------------------------------------------------------- -+ -+ - Using ccache with other compiler wrappers - ----------------------------------------- - -diff --git a/Makefile.in b/Makefile.in -index 5aee02d..08b3633 100644 ---- a/Makefile.in -+++ b/Makefile.in -@@ -37,6 +37,7 @@ non_3pp_sources = \ - lockfile.c \ - manifest.c \ - mdfour.c \ -+ memccached.c \ - stats.c \ - unify.c \ - util.c \ -@@ -101,7 +102,7 @@ perf: ccache$(EXEEXT) - .PHONY: test - test: ccache$(EXEEXT) test/main$(EXEEXT) - test/main$(EXEEXT) -- CC='$(CC)' $(srcdir)/test.sh -+ CC='$(CC)' @ccache_memcached@$(srcdir)/test.sh - - .PHONY: quicktest - quicktest: test/main$(EXEEXT) -diff --git a/ccache.c b/ccache.c -index 88e0ec5..12026c7 100644 ---- a/ccache.c -+++ b/ccache.c -@@ -102,6 +102,9 @@ static char *output_dia = NULL; - // Split dwarf information (GCC 4.8 andup). Contains pathname if not NULL. - static char *output_dwo = NULL; - -+// The cached key. -+static char *cached_key; -+ - // Array for storing -arch options. - #define MAX_ARCH_ARGS 10 - static size_t arch_args_size = 0; -@@ -123,6 +126,9 @@ static char *cached_stderr; - // (cachedir/a/b/cdef[...]-size.d). - static char *cached_dep; - -+// The manifest key. -+static char *manifest_name; -+ - // Full path to the file containing the coverage information - // (cachedir/a/b/cdef[...]-size.gcno). - static char *cached_cov; -@@ -239,6 +245,18 @@ static pid_t compiler_pid = 0; - // stored in the cache changes in a backwards-incompatible way. - static const char HASH_PREFIX[] = "3"; - -+static void from_fscache(enum fromcache_call_mode mode, -+ bool put_object_in_manifest); -+static void to_fscache(struct args *args); -+#ifdef HAVE_LIBMEMCACHED -+static void from_memcached(enum fromcache_call_mode mode, -+ bool put_object_in_manifest); -+static void to_memcached(struct args *args); -+#endif -+static void (*from_cache)(enum fromcache_call_mode mode, -+ bool put_object_in_manifest); -+static void (*to_cache)(struct args *args); -+ - static void - add_prefix(struct args *args, char *prefix_command) - { -@@ -952,6 +970,28 @@ put_file_in_cache(const char *source, const char *dest) - stats_update_size(file_size(&st), 1); - } - -+#ifdef HAVE_LIBMEMCACHED -+// Copy data to the cache. -+static void -+put_data_in_cache(void *data, size_t size, const char *dest) -+{ -+ int ret; -+ -+ assert(!conf->read_only); -+ assert(!conf->read_only_direct); -+ -+ /* already compressed (in cache) */ -+ ret = write_file(data, dest, size); -+ if (ret != 0) { -+ cc_log("Failed to write to %s: %s", dest, strerror(errno)); -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ cc_log("Stored in cache: %zu bytes -> %s", size, dest); -+ stats_update_size(size, 1); -+} -+#endif -+ - // Copy or link a file from the cache. - static void - get_file_from_cache(const char *source, const char *dest) -@@ -1006,6 +1046,11 @@ send_cached_stderr(void) - // Create or update the manifest file. - void update_manifest_file(void) - { -+#ifdef HAVE_LIBMEMCACHED -+ char *data; -+ size_t size; -+#endif -+ - if (!conf->direct_mode - || !included_files - || conf->read_only -@@ -1023,6 +1068,14 @@ void update_manifest_file(void) - update_mtime(manifest_path); - if (x_stat(manifest_path, &st) == 0) { - stats_update_size(file_size(&st) - old_size, old_size == 0 ? 1 : 0); -+#if HAVE_LIBMEMCACHED -+ if (strlen(conf->memcached_conf) > 0 && !conf->read_only_memcached && -+ read_file(manifest_path, st.st_size, &data, &size)) { -+ cc_log("Storing %s in memcached", manifest_name); -+ memccached_raw_set(manifest_name, data, size); -+ free(data); -+ } -+#endif - } - } else { - cc_log("Failed to add object file hash to %s", manifest_path); -@@ -1031,8 +1084,12 @@ void update_manifest_file(void) - - // Run the real compiler and put the result in cache. - static void --to_cache(struct args *args) -+to_fscache(struct args *args) - { -+#ifdef HAVE_LIBMEMCACHED -+ char *data_obj, *data_stderr, *data_dia, *data_dep; -+ size_t size_obj, size_stderr, size_dia, size_dep; -+#endif - char *tmp_stdout = format("%s.tmp.stdout", cached_obj); - int tmp_stdout_fd = create_tmp_fd(&tmp_stdout); - char *tmp_stderr = format("%s.tmp.stderr", cached_obj); -@@ -1288,6 +1345,40 @@ to_cache(struct args *args) - } - } - -+#ifdef HAVE_LIBMEMCACHED -+ if (strlen(conf->memcached_conf) > 0 && !conf->read_only_memcached && -+ !using_split_dwarf && /* no support for the dwo files just yet */ -+ !generating_coverage) { /* coverage refers to local paths anyway */ -+ cc_log("Storing %s in memcached", cached_key); -+ if (!read_file(cached_obj, 0, &data_obj, &size_obj)) { -+ data_obj = NULL; -+ size_obj = 0; -+ } -+ if (!read_file(cached_stderr, 0, &data_stderr, &size_stderr)) { -+ data_stderr = NULL; -+ size_stderr = 0; -+ } -+ if (!read_file(cached_dia, 0, &data_dia, &size_dia)) { -+ data_dia = NULL; -+ size_dia = 0; -+ } -+ if (!read_file(cached_dep, 0, &data_dep, &size_dep)) { -+ data_dep = NULL; -+ size_dep = 0; -+ } -+ -+ if (data_obj) { -+ memccached_set(cached_key, -+ data_obj, data_stderr, data_dia, data_dep, -+ size_obj, size_stderr, size_dia, size_dep); -+ } -+ -+ free(data_obj); -+ free(data_stderr); -+ free(data_dia); -+ free(data_dep); -+ } -+#endif - // Everything OK. - send_cached_stderr(); - update_manifest_file(); -@@ -1298,6 +1389,226 @@ to_cache(struct args *args) - free(tmp_dwo); - } - -+#ifdef HAVE_LIBMEMCACHED -+// Run the real compiler and put the result in cache. -+static void -+to_memcached(struct args *args) -+{ -+ const char *tmp_dir = temp_dir(); -+ char *tmp_stdout, *tmp_stderr; -+ char *stderr_d, *obj_d, *dia_d = NULL, *dep_d = NULL; -+ size_t stderr_l = 0, obj_l = 0, dia_l = 0, dep_l = 0; -+ struct stat st; -+ int status, tmp_stdout_fd, tmp_stderr_fd; -+ -+ tmp_stdout = format("%s/%s.tmp.stdout.%s", tmp_dir, cached_obj, tmp_string()); -+ tmp_stdout_fd = create_tmp_fd(&tmp_stdout); -+ tmp_stderr = format("%s/%s.tmp.stderr.%s", tmp_dir, cached_obj, tmp_string()); -+ tmp_stderr_fd = create_tmp_fd(&tmp_stderr); -+ -+ if (generating_coverage) { -+ cc_log("No memcached support for coverage yet"); -+ failed(); -+ } -+ if (using_split_dwarf) { -+ cc_log("No memcached support for split dwarf yet"); -+ failed(); -+ } -+ -+ if (create_parent_dirs(tmp_stdout) != 0) { -+ fatal("Failed to create parent directory for %s: %s", -+ tmp_stdout, strerror(errno)); -+ } -+ -+ args_add(args, "-o"); -+ args_add(args, output_obj); -+ -+ if (output_dia) { -+ args_add(args, "--serialize-diagnostics"); -+ args_add(args, output_dia); -+ } -+ -+ /* Turn off DEPENDENCIES_OUTPUT when running cc1, because -+ * otherwise it will emit a line like -+ * -+ * tmp.stdout.vexed.732.o: /home/mbp/.ccache/tmp.stdout.vexed.732.i -+ */ -+ x_unsetenv("DEPENDENCIES_OUTPUT"); -+ -+ if (conf->run_second_cpp) { -+ args_add(args, input_file); -+ } else { -+ args_add(args, i_tmpfile); -+ } -+ -+ cc_log("Running real compiler"); -+ status = execute(args->argv, tmp_stdout_fd, tmp_stderr_fd, &compiler_pid); -+ args_pop(args, 3); -+ -+ if (x_stat(tmp_stdout, &st) != 0) { -+ /* The stdout file was removed - cleanup in progress? Better bail out. */ -+ stats_update(STATS_MISSING); -+ tmp_unlink(tmp_stdout); -+ tmp_unlink(tmp_stderr); -+ failed(); -+ } -+ if (st.st_size != 0) { -+ cc_log("Compiler produced stdout"); -+ stats_update(STATS_STDOUT); -+ tmp_unlink(tmp_stdout); -+ tmp_unlink(tmp_stderr); -+ failed(); -+ } -+ tmp_unlink(tmp_stdout); -+ -+ /* -+ * Merge stderr from the preprocessor (if any) and stderr from the real -+ * compiler into tmp_stderr. -+ */ -+ if (cpp_stderr) { -+ int fd_cpp_stderr; -+ int fd_real_stderr; -+ int fd_result; -+ char *tmp_stderr2; -+ -+ tmp_stderr2 = format("%s.2", tmp_stderr); -+ if (x_rename(tmp_stderr, tmp_stderr2)) { -+ cc_log("Failed to rename %s to %s: %s", tmp_stderr, tmp_stderr2, -+ strerror(errno)); -+ failed(); -+ } -+ fd_cpp_stderr = open(cpp_stderr, O_RDONLY | O_BINARY); -+ if (fd_cpp_stderr == -1) { -+ cc_log("Failed opening %s: %s", cpp_stderr, strerror(errno)); -+ failed(); -+ } -+ fd_real_stderr = open(tmp_stderr2, O_RDONLY | O_BINARY); -+ if (fd_real_stderr == -1) { -+ cc_log("Failed opening %s: %s", tmp_stderr2, strerror(errno)); -+ failed(); -+ } -+ fd_result = open(tmp_stderr, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666); -+ if (fd_result == -1) { -+ cc_log("Failed opening %s: %s", tmp_stderr, strerror(errno)); -+ failed(); -+ } -+ copy_fd(fd_cpp_stderr, fd_result); -+ copy_fd(fd_real_stderr, fd_result); -+ close(fd_cpp_stderr); -+ close(fd_real_stderr); -+ close(fd_result); -+ tmp_unlink(tmp_stderr2); -+ free(tmp_stderr2); -+ } -+ -+ if (status != 0) { -+ int fd; -+ cc_log("Compiler gave exit status %d", status); -+ stats_update(STATS_STATUS); -+ -+ fd = open(tmp_stderr, O_RDONLY | O_BINARY); -+ if (fd != -1) { -+ /* We can output stderr immediately instead of rerunning the compiler. */ -+ copy_fd(fd, 2); -+ close(fd); -+ tmp_unlink(tmp_stderr); -+ -+ x_exit(status); -+ } -+ -+ tmp_unlink(tmp_stderr); -+ failed(); -+ } -+ -+ if (stat(output_obj, &st) != 0) { -+ cc_log("Compiler didn't produce an object file"); -+ stats_update(STATS_NOOUTPUT); -+ failed(); -+ } -+ if (st.st_size == 0) { -+ cc_log("Compiler produced an empty object file"); -+ stats_update(STATS_EMPTYOUTPUT); -+ failed(); -+ } -+ -+ if (x_stat(tmp_stderr, &st) != 0) { -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ /* cache stderr */ -+ if (!read_file(tmp_stderr, 0, &stderr_d, &stderr_l)) { -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ tmp_unlink(tmp_stderr); -+ -+ if (output_dia) { -+ if (x_stat(output_dia, &st) != 0) { -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ /* cache dia */ -+ if (!read_file(output_dia, 0, &dia_d, &dia_l)) { -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ } -+ -+ /* cache output */ -+ if (!read_file(output_obj, 0, &obj_d, &obj_l)) { -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ -+ if (generating_dependencies) { -+ if (!read_file(output_dep, 0, &dep_d, &dep_l)) { -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ } -+ -+ if (memccached_set(cached_key, obj_d, stderr_d, dia_d, dep_d, -+ obj_l, stderr_l, dia_l, dep_l) < 0) { -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ -+ cc_log("Storing %s in memcached", cached_key); -+ -+ stats_update(STATS_TOCACHE); -+ -+ /* Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can -+ * be done almost anywhere, but we might as well do it near the end as we -+ * save the stat call if we exit early. -+ */ -+ { -+ char *first_level_dir = dirname(stats_file); -+ if (create_cachedirtag(first_level_dir) != 0) { -+ cc_log("Failed to create %s/CACHEDIR.TAG (%s)\n", -+ first_level_dir, strerror(errno)); -+ stats_update(STATS_ERROR); -+ failed(); -+ } -+ free(first_level_dir); -+ -+ /* Remove any CACHEDIR.TAG on the cache_dir level where it was located in -+ * previous ccache versions. */ -+ if (getpid() % 1000 == 0) { -+ char *path = format("%s/CACHEDIR.TAG", conf->cache_dir); -+ x_unlink(path); -+ free(path); -+ } -+ } -+ -+ /* Everything OK. */ -+ send_cached_stderr(); -+ update_manifest_file(); -+ -+ free(tmp_stderr); -+ free(tmp_stdout); -+} -+#endif -+ - // Find the object file name by running the compiler in preprocessor mode. - // Returns the hash as a heap-allocated hex string. - static struct file_hash * -@@ -1408,6 +1719,7 @@ static void - update_cached_result_globals(struct file_hash *hash) - { - char *object_name = format_hash_as_string(hash->hash, hash->size); -+ cached_key = strdup(object_name); - cached_obj_hash = hash; - cached_obj = get_path_in_cache(object_name, ".o"); - cached_stderr = get_path_in_cache(object_name, ".stderr"); -@@ -1599,6 +1911,11 @@ calculate_common_hash(struct args *args, struct mdfour *hash) - static struct file_hash * - calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) - { -+#if HAVE_LIBMEMCACHED -+ char *data; -+ size_t size; -+#endif -+ - if (direct_mode) { - hash_delimiter(hash, "manifest version"); - hash_int(hash, MANIFEST_VERSION); -@@ -1791,7 +2108,27 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) - } - char *manifest_name = hash_result(hash); - manifest_path = get_path_in_cache(manifest_name, ".manifest"); -- free(manifest_name); -+ /* Check if the manifest file is there. */ -+ struct stat st; -+ if (stat(manifest_path, &st) != 0) { -+#if HAVE_LIBMEMCACHED -+ void *cache = NULL; -+#endif -+ cc_log("Manifest file %s not in cache", manifest_path); -+#if HAVE_LIBMEMCACHED -+ if (strlen(conf->memcached_conf) > 0) { -+ cc_log("Getting %s from memcached", manifest_name); -+ cache = memccached_raw_get(manifest_name, &data, &size); -+ } -+ if (cache) { -+ cc_log("Added object file hash to %s", manifest_path); -+ write_file(data, manifest_path, size); -+ stats_update_size(size, 1); -+ free(cache); -+ } else -+#endif -+ return NULL; -+ } - cc_log("Looking for object file hash in %s", manifest_path); - object_hash = manifest_get(conf, manifest_path); - if (object_hash) { -@@ -1828,8 +2165,13 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) - // Try to return the compile result from cache. If we can return from cache - // then this function exits with the correct status code, otherwise it returns. - static void --from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) -+from_fscache(enum fromcache_call_mode mode, bool put_object_in_manifest) - { -+#if HAVE_LIBMEMCACHED -+ char *data_obj, *data_stderr, *data_dia, *data_dep; -+ size_t size_obj, size_stderr, size_dia, size_dep; -+#endif -+ - // The user might be disabling cache hits. - if (conf->recache) { - return; -@@ -1837,7 +2179,33 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) - - struct stat st; - if (stat(cached_obj, &st) != 0) { -+#if HAVE_LIBMEMCACHED -+ void *cache = NULL; -+#endif - cc_log("Object file %s not in cache", cached_obj); -+#if HAVE_LIBMEMCACHED -+ if (strlen(conf->memcached_conf) > 0 && -+ !using_split_dwarf && -+ !generating_coverage) { -+ cc_log("Getting %s from memcached", cached_key); -+ cache = memccached_get(cached_key, -+ &data_obj, &data_stderr, &data_dia, &data_dep, -+ &size_obj, &size_stderr, &size_dia, &size_dep); -+ } -+ if (cache) { -+ put_data_in_cache(data_obj, size_obj, cached_obj); -+ if (size_stderr > 0) { -+ put_data_in_cache(data_stderr, size_stderr, cached_stderr); -+ } -+ if (size_dia > 0) { -+ put_data_in_cache(data_dia, size_dia, cached_dia); -+ } -+ if (size_dep > 0) { -+ put_data_in_cache(data_dep, size_dep, cached_dep); -+ } -+ memccached_free(cache); -+ } else -+#endif - return; - } - -@@ -1947,6 +2315,97 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) - x_exit(0); - } - -+#ifdef HAVE_LIBMEMCACHED -+/* -+ * Try to return the compile result from cache. If we can return from cache -+ * then this function exits with the correct status code, otherwise it returns. -+ */ -+static void -+from_memcached(enum fromcache_call_mode mode, bool put_object_in_manifest) -+{ -+ bool produce_dep_file = false; -+ int ret; -+ void *cache; -+ char *data_obj, *data_stderr, *data_dia, *data_dep; -+ size_t size_obj, size_stderr, size_dia, size_dep; -+ -+ /* the user might be disabling cache hits */ -+ if (conf->recache || using_split_dwarf || generating_coverage) { -+ return; -+ } -+ -+ cc_log("Getting %s from memcached", cached_key); -+ cache = memccached_get(cached_key, -+ &data_obj, &data_stderr, &data_dia, &data_dep, -+ &size_obj, &size_stderr, &size_dia, &size_dep); -+ if (!cache) { -+ return; -+ } -+ -+ /* -+ * (If mode != FROMCACHE_DIRECT_MODE, the dependency file is created by -+ * gcc.) -+ */ -+ produce_dep_file = generating_dependencies && mode == FROMCACHE_DIRECT_MODE; -+ -+ if (!str_eq(output_obj, "/dev/null")) { -+ x_unlink(output_obj); -+ ret = write_file(data_obj, output_obj, size_obj); -+ } else { -+ ret = 0; -+ } -+ if (ret < 0) { -+ cc_log("Problem creating %s from %s", output_obj, cached_key); -+ failed(); -+ } -+ -+ if (produce_dep_file) { -+ x_unlink(output_dep); -+ ret = write_file(data_dep, output_dep, size_dep); -+ if (ret < 0) { -+ cc_log("Problem creating %s from %s", output_dep, cached_key); -+ failed(); -+ } -+ } -+ if (output_dia) { -+ x_unlink(output_dia); -+ ret = write_file(data_dia, output_dia, size_dia); -+ if (ret < 0) { -+ cc_log("Problem creating %s from %s", output_dia, cached_key); -+ failed(); -+ } -+ } -+ -+ if (generating_dependencies && mode == FROMCACHE_CPP_MODE) { -+ /* Store the dependency file in the cache. */ -+ cc_log("Does not support non direct mode"); -+ } -+ -+ /* Send the stderr, if any. */ -+ safe_write(2, data_stderr, size_stderr); -+ -+ if (put_object_in_manifest) { -+ update_manifest_file(); -+ } -+ -+ /* log the cache hit */ -+ switch (mode) { -+ case FROMCACHE_DIRECT_MODE: -+ cc_log("Succeeded getting cached result"); -+ stats_update(STATS_CACHEHIT_DIR); -+ break; -+ -+ case FROMCACHE_CPP_MODE: -+ cc_log("Succeeded getting cached result"); -+ stats_update(STATS_CACHEHIT_CPP); -+ break; -+ } -+ -+ /* and exit with the right status code */ -+ x_exit(0); -+} -+#endif -+ - // Find the real compiler. We just search the PATH to find an executable of the - // same name that isn't a link to ourselves. - static void -@@ -3059,6 +3518,19 @@ initialize(void) - create_initial_config_file(conf, primary_config_path); - } - -+ from_cache = from_fscache; -+ to_cache = to_fscache; -+ -+#ifdef HAVE_LIBMEMCACHED -+ if (strlen(conf->memcached_conf) > 0) { -+ memccached_init(conf->memcached_conf); -+ } -+ -+ if (conf->memcached_only) { -+ from_cache = from_memcached; -+ to_cache = to_memcached; -+ } -+#endif - exitfn_init(); - exitfn_add_nullary(stats_flush); - exitfn_add_nullary(clean_up_pending_tmp_files); -@@ -3089,6 +3561,7 @@ cc_reset(void) - free(output_dep); output_dep = NULL; - free(output_cov); output_cov = NULL; - free(output_dia); output_dia = NULL; -+ free(cached_key); cached_key = NULL; - free(cached_obj_hash); cached_obj_hash = NULL; - free(cached_obj); cached_obj = NULL; - free(cached_dwo); cached_dwo = NULL; -@@ -3096,6 +3569,7 @@ cc_reset(void) - free(cached_dep); cached_dep = NULL; - free(cached_cov); cached_cov = NULL; - free(cached_dia); cached_dia = NULL; -+ free(manifest_name); manifest_name = NULL; - free(manifest_path); manifest_path = NULL; - time_of_compilation = 0; - for (size_t i = 0; i < ignore_headers_len; i++) { -@@ -3119,6 +3593,10 @@ cc_reset(void) - free(stats_file); stats_file = NULL; - output_is_precompiled_header = false; - -+#ifdef HAVE_LIBMEMCACHED -+ memccached_release(); -+#endif -+ - conf = conf_create(); - using_split_dwarf = false; - } -@@ -3285,8 +3763,14 @@ ccache(int argc, char *argv[]) - put_object_in_manifest = true; - } - -- // If we can return from cache at this point then do. -- from_cache(FROMCACHE_CPP_MODE, put_object_in_manifest); -+ /* don't hit memcached twice */ -+ if (conf->memcached_only && object_hash_from_manifest -+ && file_hashes_equal(object_hash_from_manifest, object_hash)) { -+ cc_log("Already searched for %s", cached_key); -+ } else { -+ // If we can return from cache at this point then do. -+ from_cache(FROMCACHE_CPP_MODE, put_object_in_manifest); -+ } - - if (conf->read_only) { - cc_log("Read-only mode; running real compiler"); -diff --git a/ccache.h b/ccache.h -index 7b29bb8..1c1e38d 100644 ---- a/ccache.h -+++ b/ccache.h -@@ -126,6 +126,8 @@ void cc_log_argv(const char *prefix, char **argv); - void fatal(const char *format, ...) ATTR_FORMAT(printf, 1, 2) ATTR_NORETURN; - - void copy_fd(int fd_in, int fd_out); -+int safe_write(int fd_out, const char *data, size_t length); -+int write_file(const char *data, const char *dest, size_t length); - int copy_file(const char *src, const char *dest, int compress_level); - int move_file(const char *src, const char *dest, int compress_level); - int move_uncompressed_file(const char *src, const char *dest, -@@ -185,6 +187,23 @@ char *read_text_file(const char *path, size_t size_hint); - char *subst_env_in_string(const char *str, char **errmsg); - - // ---------------------------------------------------------------------------- -+// memccached.c -+ -+int memccached_init(char *conf); -+int memccached_raw_set(const char *key, const char* data, size_t len); -+int memccached_set( -+ const char *key, -+ const char *out, const char *err, const char *dia, const char *dep, -+ size_t out_len, size_t err_len, size_t dia_len, size_t dep_len); -+void *memccached_raw_get(const char *key, char **data, size_t *len); -+void* memccached_get( -+ const char *key, -+ char **out, char **err, char **dia, char **dep, -+ size_t *out_len, size_t *err_len, size_t *dia_len, size_t *dep_len); -+void memccached_free(void *blob); -+int memccached_release(void); -+ -+// ---------------------------------------------------------------------------- - // stats.c - - void stats_update(enum stats stat); -diff --git a/conf.c b/conf.c -index cfa2874..bf4e365 100644 ---- a/conf.c -+++ b/conf.c -@@ -329,11 +329,14 @@ conf_create(void) - conf->log_file = x_strdup(""); - conf->max_files = 0; - conf->max_size = (uint64_t)5 * 1000 * 1000 * 1000; -+ conf->memcached_conf = x_strdup(""); -+ conf->memcached_only = false; - conf->path = x_strdup(""); - conf->prefix_command = x_strdup(""); - conf->prefix_command_cpp = x_strdup(""); - conf->read_only = false; - conf->read_only_direct = false; -+ conf->read_only_memcached = false; - conf->recache = false; - conf->run_second_cpp = true; - conf->sloppiness = 0; -@@ -362,6 +365,7 @@ conf_free(struct conf *conf) - free(conf->extra_files_to_hash); - free(conf->ignore_headers_in_manifest); - free(conf->log_file); -+ free(conf->memcached_conf); - free(conf->path); - free(conf->prefix_command); - free(conf->prefix_command_cpp); -@@ -594,6 +598,12 @@ conf_print_items(struct conf *conf, - printer(s, conf->item_origins[find_conf("max_size")->number], context); - free(s2); - -+ reformat(&s, "memcached_conf = %s", conf->memcached_conf); -+ printer(s, conf->item_origins[find_conf("memcached_conf")->number], context); -+ -+ reformat(&s, "memcached_only = %s", bool_to_string(conf->memcached_only)); -+ printer(s, conf->item_origins[find_conf("memcached_only")->number], context); -+ - reformat(&s, "path = %s", conf->path); - printer(s, conf->item_origins[find_conf("path")->number], context); - -@@ -611,6 +621,11 @@ conf_print_items(struct conf *conf, - printer(s, conf->item_origins[find_conf("read_only_direct")->number], - context); - -+ reformat(&s, "read_only_memcached = %s", -+ bool_to_string(conf->read_only_memcached)); -+ printer(s, conf->item_origins[find_conf("read_only_memcached")->number], -+ context); -+ - reformat(&s, "recache = %s", bool_to_string(conf->recache)); - printer(s, conf->item_origins[find_conf("recache")->number], context); - -diff --git a/conf.h b/conf.h -index 232dcfd..1e22016 100644 ---- a/conf.h -+++ b/conf.h -@@ -23,11 +23,14 @@ struct conf { - char *log_file; - unsigned max_files; - uint64_t max_size; -+ char *memcached_conf; -+ bool memcached_only; - char *path; - char *prefix_command; - char *prefix_command_cpp; - bool read_only; - bool read_only_direct; -+ bool read_only_memcached; - bool recache; - bool run_second_cpp; - unsigned sloppiness; -diff --git a/configure.ac b/configure.ac -index a35fac0..7ef33e1 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -16,6 +16,7 @@ case $host in - ;; - esac - -+AC_SUBST(ccache_memcached) - AC_SUBST(extra_libs) - AC_SUBST(include_dev_mk) - AC_SUBST(test_suites) -@@ -84,6 +85,31 @@ HW_FUNC_ASPRINTF - dnl Check if -lm is needed. - AC_SEARCH_LIBS(cos, m) - -+AC_ARG_ENABLE(static, -+ [AS_HELP_STRING([--enable-static], -+ [enable static link])]) -+ -+if test x${enable_static} != x; then -+ extra_ldflags="-static" -+fi -+ -+AC_ARG_ENABLE(memcached, -+ [AS_HELP_STRING([--enable-memcached], -+ [enable memcached as a cache backend])]) -+ -+dnl enable-memcached: Check if -lmemcached is needed. -+if test x${enable_memcached} != x; then -+ if test x${enable_static} != x; then -+ AC_CHECK_LIB(stdc++, __gxx_personality_v0,[]) -+ fi -+ AC_CHECK_LIB(pthread, pthread_once) -+ AC_CHECK_LIB(memcached, memcached,[],[ -+ echo ' WARNING: recent version libmemcached not found' -+ echo ' please install libmemcached > 1.0 with development files' -+ exit 1 -+ ]) -+ ccache_memcached='CCACHE_MEMCACHED=1 ' -+fi - - dnl Check for zlib - AC_ARG_WITH(bundled-zlib, -diff --git a/confitems.gperf b/confitems.gperf -index 531bc92..fd43765 100644 ---- a/confitems.gperf -+++ b/confitems.gperf -@@ -26,15 +26,18 @@ limit_multiple, 15, ITEM(limit_multiple, float) - log_file, 16, ITEM(log_file, env_string) - max_files, 17, ITEM(max_files, unsigned) - max_size, 18, ITEM(max_size, size) --path, 19, ITEM(path, env_string) --prefix_command, 20, ITEM(prefix_command, env_string) --prefix_command_cpp, 21, ITEM(prefix_command_cpp, env_string) --read_only, 22, ITEM(read_only, bool) --read_only_direct, 23, ITEM(read_only_direct, bool) --recache, 24, ITEM(recache, bool) --run_second_cpp, 25, ITEM(run_second_cpp, bool) --sloppiness, 26, ITEM(sloppiness, sloppiness) --stats, 27, ITEM(stats, bool) --temporary_dir, 28, ITEM(temporary_dir, env_string) --umask, 29, ITEM(umask, umask) --unify, 30, ITEM(unify, bool) -+memcached_conf, 19, ITEM(memcached_conf, string) -+memcached_only, 20, ITEM(memcached_only, bool) -+path, 21, ITEM(path, env_string) -+prefix_command, 22, ITEM(prefix_command, env_string) -+prefix_command_cpp, 23, ITEM(prefix_command_cpp, env_string) -+read_only, 24, ITEM(read_only, bool) -+read_only_direct, 25, ITEM(read_only_direct, bool) -+read_only_memcached, 26, ITEM(read_only_memcached, bool) -+recache, 27, ITEM(recache, bool) -+run_second_cpp, 28, ITEM(run_second_cpp, bool) -+sloppiness, 29, ITEM(sloppiness, sloppiness) -+stats, 30, ITEM(stats, bool) -+temporary_dir, 31, ITEM(temporary_dir, env_string) -+umask, 32, ITEM(umask, umask) -+unify, 33, ITEM(unify, bool) -diff --git a/confitems_lookup.c b/confitems_lookup.c -index 7482557..b324dad 100644 ---- a/confitems_lookup.c -+++ b/confitems_lookup.c -@@ -1,6 +1,6 @@ - /* ANSI-C code produced by gperf version 3.0.4 */ - /* Command-line: gperf confitems.gperf */ --/* Computed positions: -k'1-2' */ -+/* Computed positions: -k'1,$' */ - - #if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \ - && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \ -@@ -31,7 +31,7 @@ - - #line 8 "confitems.gperf" - struct conf_item; --/* maximum key range = 46, duplicates = 0 */ -+/* maximum key range = 65, duplicates = 0 */ - - #ifdef __GNUC__ - __inline -@@ -45,34 +45,34 @@ confitems_hash (register const char *str, register unsigned int len) - { - static const unsigned char asso_values[] = - { -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 0, 13, 0, -- 5, 10, 50, 0, 30, 20, 50, 0, 10, 20, -- 5, 0, 0, 50, 5, 0, 10, 15, 50, 50, -- 20, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, -- 50, 50, 50, 50, 50, 50 -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 5, 20, -+ 5, 0, 30, 70, 30, 10, 70, 20, 25, 0, -+ 10, 70, 0, 70, 0, 0, 10, 0, 70, 70, -+ 70, 55, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, -+ 70, 70, 70, 70, 70, 70 - }; -- return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]]; -+ return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]]; - } - - static -@@ -87,91 +87,108 @@ confitems_get (register const char *str, register unsigned int len) - { - enum - { -- TOTAL_KEYWORDS = 31, -+ TOTAL_KEYWORDS = 34, - MIN_WORD_LENGTH = 4, - MAX_WORD_LENGTH = 26, -- MIN_HASH_VALUE = 4, -- MAX_HASH_VALUE = 49 -+ MIN_HASH_VALUE = 5, -+ MAX_HASH_VALUE = 69 - }; - - static const struct conf_item wordlist[] = - { - {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, - {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, --#line 29 "confitems.gperf" -- {"path", 19, ITEM(path, env_string)}, -- {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, - {"",0,NULL,0,NULL}, --#line 13 "confitems.gperf" -- {"compiler", 3, ITEM(compiler, string)}, --#line 11 "confitems.gperf" -- {"cache_dir", 1, ITEM(cache_dir, env_string)}, -- {"",0,NULL,0,NULL}, --#line 15 "confitems.gperf" -- {"compression", 5, ITEM(compression, bool)}, -- {"",0,NULL,0,NULL}, --#line 17 "confitems.gperf" -- {"cpp_extension", 7, ITEM(cpp_extension, string)}, --#line 14 "confitems.gperf" -- {"compiler_check", 4, ITEM(compiler_check, string)}, --#line 37 "confitems.gperf" -- {"stats", 27, ITEM(stats, bool)}, --#line 12 "confitems.gperf" -- {"cache_dir_levels", 2, ITEM_V(cache_dir_levels, unsigned, dir_levels)}, --#line 16 "confitems.gperf" -- {"compression_level", 6, ITEM(compression_level, unsigned)}, --#line 26 "confitems.gperf" -- {"log_file", 16, ITEM(log_file, env_string)}, --#line 30 "confitems.gperf" -- {"prefix_command", 20, ITEM(prefix_command, env_string)}, --#line 36 "confitems.gperf" -- {"sloppiness", 26, ITEM(sloppiness, sloppiness)}, --#line 10 "confitems.gperf" -- {"base_dir", 0, ITEM_V(base_dir, env_string, absolute_path)}, --#line 34 "confitems.gperf" -- {"recache", 24, ITEM(recache, bool)}, --#line 31 "confitems.gperf" -- {"prefix_command_cpp", 21, ITEM(prefix_command_cpp, env_string)}, --#line 32 "confitems.gperf" -- {"read_only", 22, ITEM(read_only, bool)}, - #line 40 "confitems.gperf" -- {"unify", 30, ITEM(unify, bool)}, -+ {"stats", 30, ITEM(stats, bool)}, - {"",0,NULL,0,NULL}, --#line 24 "confitems.gperf" -- {"keep_comments_cpp", 14, ITEM(keep_comments_cpp, bool)}, -+#line 37 "confitems.gperf" -+ {"recache", 27, ITEM(recache, bool)}, - #line 28 "confitems.gperf" - {"max_size", 18, ITEM(max_size, size)}, - #line 27 "confitems.gperf" - {"max_files", 17, ITEM(max_files, unsigned)}, -+#line 39 "confitems.gperf" -+ {"sloppiness", 29, ITEM(sloppiness, sloppiness)}, - {"",0,NULL,0,NULL}, --#line 33 "confitems.gperf" -- {"read_only_direct", 23, ITEM(read_only_direct, bool)}, - #line 19 "confitems.gperf" - {"disable", 9, ITEM(disable, bool)}, -+#line 10 "confitems.gperf" -+ {"base_dir", 0, ITEM_V(base_dir, env_string, absolute_path)}, - #line 38 "confitems.gperf" -- {"temporary_dir", 28, ITEM(temporary_dir, env_string)}, --#line 35 "confitems.gperf" -- {"run_second_cpp", 25, ITEM(run_second_cpp, bool)}, -+ {"run_second_cpp", 28, ITEM(run_second_cpp, bool)}, - {"",0,NULL,0,NULL}, - #line 18 "confitems.gperf" - {"direct_mode", 8, ITEM(direct_mode, bool)}, - {"",0,NULL,0,NULL}, --#line 22 "confitems.gperf" -- {"hash_dir", 12, ITEM(hash_dir, bool)}, --#line 21 "confitems.gperf" -- {"hard_link", 11, ITEM(hard_link, bool)}, --#line 39 "confitems.gperf" -- {"umask", 29, ITEM(umask, umask)}, -+#line 33 "confitems.gperf" -+ {"prefix_command_cpp", 23, ITEM(prefix_command_cpp, env_string)}, -+#line 32 "confitems.gperf" -+ {"prefix_command", 22, ITEM(prefix_command, env_string)}, - {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, - {"",0,NULL,0,NULL}, -+#line 41 "confitems.gperf" -+ {"temporary_dir", 31, ITEM(temporary_dir, env_string)}, -+#line 36 "confitems.gperf" -+ {"read_only_memcached", 26, ITEM(read_only_memcached, bool)}, -+#line 42 "confitems.gperf" -+ {"umask", 32, ITEM(umask, umask)}, -+#line 35 "confitems.gperf" -+ {"read_only_direct", 25, ITEM(read_only_direct, bool)}, -+ {"",0,NULL,0,NULL}, -+#line 13 "confitems.gperf" -+ {"compiler", 3, ITEM(compiler, string)}, -+#line 11 "confitems.gperf" -+ {"cache_dir", 1, ITEM(cache_dir, env_string)}, -+ {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -+ {"",0,NULL,0,NULL}, -+#line 26 "confitems.gperf" -+ {"log_file", 16, ITEM(log_file, env_string)}, -+#line 31 "confitems.gperf" -+ {"path", 21, ITEM(path, env_string)}, -+ {"",0,NULL,0,NULL}, -+#line 12 "confitems.gperf" -+ {"cache_dir_levels", 2, ITEM_V(cache_dir_levels, unsigned, dir_levels)}, -+#line 24 "confitems.gperf" -+ {"keep_comments_cpp", 14, ITEM(keep_comments_cpp, bool)}, -+#line 22 "confitems.gperf" -+ {"hash_dir", 12, ITEM(hash_dir, bool)}, - #line 25 "confitems.gperf" - {"limit_multiple", 15, ITEM(limit_multiple, float)}, - {"",0,NULL,0,NULL}, -+#line 15 "confitems.gperf" -+ {"compression", 5, ITEM(compression, bool)}, -+ {"",0,NULL,0,NULL}, -+#line 17 "confitems.gperf" -+ {"cpp_extension", 7, ITEM(cpp_extension, string)}, -+#line 29 "confitems.gperf" -+ {"memcached_conf", 19, ITEM(memcached_conf, string)}, -+ {"",0,NULL,0,NULL}, - #line 23 "confitems.gperf" - {"ignore_headers_in_manifest", 13, ITEM(ignore_headers_in_manifest, env_string)}, - {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, - #line 20 "confitems.gperf" -- {"extra_files_to_hash", 10, ITEM(extra_files_to_hash, env_string)} -+ {"extra_files_to_hash", 10, ITEM(extra_files_to_hash, env_string)}, -+ {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -+ {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -+#line 14 "confitems.gperf" -+ {"compiler_check", 4, ITEM(compiler_check, string)}, -+ {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -+ {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -+#line 21 "confitems.gperf" -+ {"hard_link", 11, ITEM(hard_link, bool)}, -+#line 43 "confitems.gperf" -+ {"unify", 33, ITEM(unify, bool)}, -+ {"",0,NULL,0,NULL}, -+#line 16 "confitems.gperf" -+ {"compression_level", 6, ITEM(compression_level, unsigned)}, -+ {"",0,NULL,0,NULL}, -+#line 34 "confitems.gperf" -+ {"read_only", 24, ITEM(read_only, bool)}, -+ {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -+ {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -+#line 30 "confitems.gperf" -+ {"memcached_only", 20, ITEM(memcached_only, bool)} - }; - - if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH) -@@ -188,4 +205,4 @@ confitems_get (register const char *str, register unsigned int len) - } - return 0; - } --static const size_t CONFITEMS_TOTAL_KEYWORDS = 31; -+static const size_t CONFITEMS_TOTAL_KEYWORDS = 34; -diff --git a/dump-memcached.py b/dump-memcached.py -new file mode 100755 -index 0000000..e7b2b0d ---- /dev/null -+++ b/dump-memcached.py -@@ -0,0 +1,89 @@ -+#!/usr/bin/env python -+ -+import memcache -+import struct -+import sys -+import os -+import binascii -+ -+""" -+/* blob format for storing: -+ -+ char magic[4]; # 'CCH1', might change for other version of ccache -+ # ccache will erase the blob in memcached if wrong magic -+ uint32_t obj_len; # network endian -+ char *obj[obj_len]; -+ uint32_t stderr_len; # network endian -+ char *stderr[stderr_len]; -+ uint32_t dia_len; # network endian -+ char *dia[dia_len]; -+ uint32_t dep_len; # network endian -+ char *dep[dep_len]; -+ -+*/ -+""" -+MEMCCACHE_MAGIC = 'CCH1' -+ -+def get_blob(token): -+ return token[4:4+struct.unpack('!I', val[0:4])[0]] -+MEMCCACHE_BIG = 'CCBM' -+ -+""" -+/* blob format for big values: -+ -+ char magic[4]; # 'CCBM' -+ uint32_t numkeys; # network endian -+ uint32_t hash_size; # network endian -+ uint32_t reserved; # network endian -+ uint32_t value_length; # network endian -+ -+ hash of include file ( bytes) -+ size of include file (4 bytes unsigned int) -+ ... -+ -+ -+ -+*/ -+""" -+MEMCCACHE_BIG = 'CCBM' -+ -+server = os.getenv("MEMCACHED_SERVERS", "localhost") -+mc = memcache.Client(server.split(','), debug=1) -+ -+key = sys.argv[1] -+val = mc.get(key) -+if val[0:4] == MEMCCACHE_BIG: -+ numkeys = struct.unpack('!I', val[4:8])[0] -+ assert struct.unpack('!I', val[8:12])[0] == 16 -+ assert struct.unpack('!I', val[12:16])[0] == 0 -+ size = struct.unpack('!I', val[16:20])[0] -+ val = val[20:] -+ buf = "" -+ while val: -+ md4 = val[0:16] -+ size = struct.unpack('!I', val[16:20])[0] -+ val = val[20:] -+ subkey = "%s-%d" % (binascii.hexlify(md4), size) -+ subval = mc.get(subkey) -+ if not subval: -+ print "%s not found" % subkey -+ buf = buf + subval -+ val = buf -+if val: -+ magic = val[0:4] -+ if magic == MEMCCACHE_MAGIC: -+ val = val[4:] -+ obj = get_blob(val) -+ val = val[4+len(obj):] -+ stderr = get_blob(val) -+ val = val[4+len(stderr):] -+ dia = get_blob(val) -+ val = val[4+len(dia):] -+ dep = get_blob(val) -+ val = val[4+len(dep):] -+ assert len(val) == 0 -+ print "%s: %d %d %d %d" % (key, len(obj), len(stderr), len(dia), len(dep)) -+ else: -+ print "wrong magic" -+else: -+ print "key missing" -diff --git a/envtoconfitems.gperf b/envtoconfitems.gperf -index 81d8444..00f64e0 100644 ---- a/envtoconfitems.gperf -+++ b/envtoconfitems.gperf -@@ -27,12 +27,15 @@ LIMIT_MULTIPLE, "limit_multiple" - LOGFILE, "log_file" - MAXFILES, "max_files" - MAXSIZE, "max_size" -+MEMCACHED_CONF, "memcached_conf" -+MEMCACHED_ONLY, "memcached_only" - NLEVELS, "cache_dir_levels" - PATH, "path" - PREFIX, "prefix_command" - PREFIX_CPP, "prefix_command_cpp" - READONLY, "read_only" - READONLY_DIRECT, "read_only_direct" -+READONLY_MEMCACHED, "read_only_memcached" - RECACHE, "recache" - SLOPPINESS, "sloppiness" - STATS, "stats" -diff --git a/envtoconfitems_lookup.c b/envtoconfitems_lookup.c -index 1265bd6..4608827 100644 ---- a/envtoconfitems_lookup.c -+++ b/envtoconfitems_lookup.c -@@ -1,6 +1,6 @@ - /* ANSI-C code produced by gperf version 3.0.4 */ - /* Command-line: gperf envtoconfitems.gperf */ --/* Computed positions: -k'1,5' */ -+/* Computed positions: -k'1,5,11' */ - - #if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \ - && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \ -@@ -31,7 +31,7 @@ - - #line 9 "envtoconfitems.gperf" - struct env_to_conf_item; --/* maximum key range = 42, duplicates = 0 */ -+/* maximum key range = 47, duplicates = 0 */ - - #ifdef __GNUC__ - __inline -@@ -45,38 +45,46 @@ envtoconfitems_hash (register const char *str, register unsigned int len) - { - static const unsigned char asso_values[] = - { -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 20, 0, 0, 10, -- 0, 44, 5, 15, 0, 44, 10, 25, 9, 0, -- 5, 10, 5, 15, 10, 5, 44, 44, 44, 44, -- 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, -- 44, 44, 44, 44, 44, 44, 44 -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 5, 0, 0, 5, -+ 40, 49, 20, 5, 0, 49, 20, 5, 0, 5, -+ 5, 0, 15, 0, 25, 0, 25, 49, 49, 49, -+ 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, -+ 49, 49, 49, 49, 49, 49, 49 - }; - register int hval = len; - - switch (hval) - { - default: -+ hval += asso_values[(unsigned char)str[10]]; -+ /*FALLTHROUGH*/ -+ case 10: -+ case 9: -+ case 8: -+ case 7: -+ case 6: -+ case 5: - hval += asso_values[(unsigned char)str[4]+1]; - /*FALLTHROUGH*/ - case 4: -@@ -101,11 +109,11 @@ envtoconfitems_get (register const char *str, register unsigned int len) - { - enum - { -- TOTAL_KEYWORDS = 31, -+ TOTAL_KEYWORDS = 34, - MIN_WORD_LENGTH = 2, -- MAX_WORD_LENGTH = 15, -+ MAX_WORD_LENGTH = 18, - MIN_HASH_VALUE = 2, -- MAX_HASH_VALUE = 43 -+ MAX_HASH_VALUE = 48 - }; - - static const struct env_to_conf_item wordlist[] = -@@ -117,72 +125,76 @@ envtoconfitems_get (register const char *str, register unsigned int len) - {"DIR", "cache_dir"}, - #line 16 "envtoconfitems.gperf" - {"CPP2", "run_second_cpp"}, -- {"",""}, -+#line 44 "envtoconfitems.gperf" -+ {"UNIFY", "unify"}, - #line 19 "envtoconfitems.gperf" - {"DIRECT", "direct_mode"}, - #line 20 "envtoconfitems.gperf" - {"DISABLE", "disable"}, --#line 17 "envtoconfitems.gperf" -- {"COMMENTS", "keep_comments_cpp"}, --#line 31 "envtoconfitems.gperf" -+#line 14 "envtoconfitems.gperf" -+ {"COMPRESS", "compression"}, -+#line 33 "envtoconfitems.gperf" - {"PATH", "path"}, --#line 41 "envtoconfitems.gperf" -- {"UNIFY", "unify"}, --#line 32 "envtoconfitems.gperf" -+#line 40 "envtoconfitems.gperf" -+ {"SLOPPINESS", "sloppiness"}, -+#line 34 "envtoconfitems.gperf" - {"PREFIX", "prefix_command"}, --#line 36 "envtoconfitems.gperf" -- {"RECACHE", "recache"}, -+#line 29 "envtoconfitems.gperf" -+ {"MAXSIZE", "max_size"}, -+#line 28 "envtoconfitems.gperf" -+ {"MAXFILES", "max_files"}, -+ {"",""}, -+#line 35 "envtoconfitems.gperf" -+ {"PREFIX_CPP", "prefix_command_cpp"}, -+ {"",""}, -+#line 11 "envtoconfitems.gperf" -+ {"BASEDIR", "base_dir"}, - #line 13 "envtoconfitems.gperf" - {"COMPILERCHECK", "compiler_check"}, -+#line 21 "envtoconfitems.gperf" -+ {"EXTENSION", "cpp_extension"}, -+#line 22 "envtoconfitems.gperf" -+ {"EXTRAFILES", "extra_files_to_hash"}, - {"",""}, --#line 33 "envtoconfitems.gperf" -- {"PREFIX_CPP", "prefix_command_cpp"}, -+#line 39 "envtoconfitems.gperf" -+ {"RECACHE", "recache"}, -+#line 25 "envtoconfitems.gperf" -+ {"IGNOREHEADERS", "ignore_headers_in_manifest"}, - #line 30 "envtoconfitems.gperf" -- {"NLEVELS", "cache_dir_levels"}, -+ {"MEMCACHED_CONF", "memcached_conf"}, -+#line 43 "envtoconfitems.gperf" -+ {"UMASK", "umask"}, -+ {"",""}, - #line 27 "envtoconfitems.gperf" - {"LOGFILE", "log_file"}, --#line 34 "envtoconfitems.gperf" -+#line 36 "envtoconfitems.gperf" - {"READONLY", "read_only"}, --#line 21 "envtoconfitems.gperf" -- {"EXTENSION", "cpp_extension"}, --#line 40 "envtoconfitems.gperf" -- {"UMASK", "umask"}, -+#line 31 "envtoconfitems.gperf" -+ {"MEMCACHED_ONLY", "memcached_only"}, -+#line 41 "envtoconfitems.gperf" -+ {"STATS", "stats"}, - {"",""}, - #line 24 "envtoconfitems.gperf" - {"HASHDIR", "hash_dir"}, --#line 14 "envtoconfitems.gperf" -- {"COMPRESS", "compression"}, -- {"",""}, --#line 35 "envtoconfitems.gperf" -- {"READONLY_DIRECT", "read_only_direct"}, -- {"",""}, --#line 39 "envtoconfitems.gperf" -+#line 23 "envtoconfitems.gperf" -+ {"HARDLINK", "hard_link"}, -+ {"",""}, {"",""}, {"",""}, -+#line 42 "envtoconfitems.gperf" - {"TEMPDIR", "temporary_dir"}, - #line 15 "envtoconfitems.gperf" - {"COMPRESSLEVEL", "compression_level"}, - #line 26 "envtoconfitems.gperf" - {"LIMIT_MULTIPLE", "limit_multiple"}, --#line 38 "envtoconfitems.gperf" -- {"STATS", "stats"}, -- {"",""}, --#line 29 "envtoconfitems.gperf" -- {"MAXSIZE", "max_size"}, --#line 28 "envtoconfitems.gperf" -- {"MAXFILES", "max_files"}, -- {"",""}, - #line 37 "envtoconfitems.gperf" -- {"SLOPPINESS", "sloppiness"}, -- {"",""}, --#line 11 "envtoconfitems.gperf" -- {"BASEDIR", "base_dir"}, --#line 23 "envtoconfitems.gperf" -- {"HARDLINK", "hard_link"}, -- {"",""}, --#line 22 "envtoconfitems.gperf" -- {"EXTRAFILES", "extra_files_to_hash"}, -+ {"READONLY_DIRECT", "read_only_direct"}, - {"",""}, {"",""}, --#line 25 "envtoconfitems.gperf" -- {"IGNOREHEADERS", "ignore_headers_in_manifest"} -+#line 38 "envtoconfitems.gperf" -+ {"READONLY_MEMCACHED", "read_only_memcached"}, -+ {"",""}, {"",""}, {"",""}, -+#line 32 "envtoconfitems.gperf" -+ {"NLEVELS", "cache_dir_levels"}, -+#line 17 "envtoconfitems.gperf" -+ {"COMMENTS", "keep_comments_cpp"} - }; - - if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH) -@@ -199,4 +211,4 @@ envtoconfitems_get (register const char *str, register unsigned int len) - } - return 0; - } --static const size_t ENVTOCONFITEMS_TOTAL_KEYWORDS = 31; -+static const size_t ENVTOCONFITEMS_TOTAL_KEYWORDS = 34; -diff --git a/memccached.c b/memccached.c -new file mode 100644 -index 0000000..38c44aa ---- /dev/null -+++ b/memccached.c -@@ -0,0 +1,433 @@ -+#include "ccache.h" -+ -+#ifdef HAVE_LIBMEMCACHED -+ -+#include -+#include -+ -+#define MEMCCACHE_MAGIC "CCH1" -+#define MEMCCACHE_BIG "CCBM" -+ -+#define MAX_VALUE_SIZE (1000 << 10) /* 1M with memcached overhead */ -+#define SPLIT_VALUE_SIZE MAX_VALUE_SIZE -+ -+/* status variables for memcached */ -+static memcached_st *memc; -+ -+int memccached_init(char *conf) -+{ -+ memc = memcached(conf, strlen(conf)); -+ if (!memc) { -+ char errorbuf[1024]; -+ libmemcached_check_configuration(conf, strlen(conf), errorbuf, 1024); -+ cc_log("Problem creating memcached with conf %s:\n%s\n", conf, errorbuf); -+ return -1; -+ } -+ /* Consistent hashing delivers better distribution and allows servers to be -+ added to the cluster with minimal cache losses */ -+ memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_DISTRIBUTION, -+ MEMCACHED_DISTRIBUTION_CONSISTENT); -+ return 0; -+} -+ -+/* blob format for big values: -+ -+ char magic[4]; # 'CCBM' -+ uint32_t numkeys; # network endian -+ uint32_t hash_size; # network endian -+ uint32_t reserved; # network endian -+ uint32_t value_length; # network endian -+ -+ hash of include file ( bytes) -+ size of include file (4 bytes unsigned int) -+ ... -+ -+ -+ -+ */ -+static memcached_return_t memccached_big_set(memcached_st *ptr, -+ const char *key, -+ size_t key_length, -+ const char *value, -+ size_t value_length, -+ time_t expiration, -+ uint32_t flags) -+{ -+ char *buf; -+ size_t buflen; -+ char *p; -+ int numkeys; -+ struct mdfour md; -+ char subkey[20]; -+ size_t n; -+ memcached_return_t ret; -+ size_t x; -+ -+ numkeys = (value_length + SPLIT_VALUE_SIZE - 1) / SPLIT_VALUE_SIZE; -+ buflen = 20 + 20 * numkeys; -+ buf = x_malloc(buflen); -+ p = buf; -+ -+ memcpy(p, MEMCCACHE_BIG, 4); -+ *((uint32_t *) (p + 4)) = htonl(numkeys); -+ *((uint32_t *) (p + 8)) = htonl(16); -+ *((uint32_t *) (p + 12)) = htonl(0); -+ *((uint32_t *) (p + 16)) = htonl(value_length); -+ p += 20; -+ -+ for (x = 0; x < value_length; x += n) { -+ size_t remain; -+ char *s; -+ -+ remain = value_length - x; -+ n = remain > SPLIT_VALUE_SIZE ? SPLIT_VALUE_SIZE : remain; -+ -+ mdfour_begin(&md); -+ mdfour_update(&md, (const unsigned char *) value + x, n); -+ mdfour_result(&md, (unsigned char *) subkey); -+ *((uint32_t *) (subkey + 16)) = htonl(n); -+ s = format_hash_as_string((const unsigned char *) subkey, n); -+ cc_log("memcached_mset %s %zu", s, n); -+ ret = memcached_set(ptr, s, strlen(s), value + x, n, -+ expiration, flags); -+ free(s); -+ if (ret) { -+ cc_log("Failed to set key in memcached: %s", -+ memcached_strerror(memc, ret)); -+ return ret; -+ } -+ -+ memcpy(p, subkey, 20); -+ p += 20; -+ } -+ -+ cc_log("memcached_set %.*s %zu (%zu)", (int) key_length, key, buflen, -+ value_length); -+ ret = memcached_set(ptr, key, key_length, buf, buflen, -+ expiration, flags); -+ free(buf); -+ return ret; -+} -+ -+static char *memccached_big_get(memcached_st *ptr, -+ const char *key, -+ size_t key_length, -+ const char *value, -+ size_t *value_length, -+ uint32_t *flags, -+ memcached_return_t *error) -+{ -+ char *buf; -+ size_t buflen; -+ size_t totalsize; -+ char *p; -+ const char *v; -+ int numkeys; -+ char **keys; -+ bool *key_seen; -+ size_t *key_lengths; -+ size_t *value_offsets; -+ int *value_lengths; -+ memcached_return_t ret; -+ memcached_result_st *result; -+ int n; -+ int i; -+ -+ if (!value) { -+ value = memcached_get(ptr, key, key_length, value_length, flags, error); -+ if (!value) { -+ return NULL; -+ } -+ } -+ -+ p = (char *) value; -+ if (memcmp(p, MEMCCACHE_BIG, 4) != 0) { -+ return NULL; -+ } -+ numkeys = ntohl(*(uint32_t *) (p + 4)); -+ assert(ntohl(*(uint32_t *) (p + 8)) == 16); -+ assert(ntohl(*(uint32_t *) (p + 12)) == 0); -+ totalsize = ntohl(*(uint32_t *) (p + 16)); -+ p += 20; -+ -+ keys = x_malloc(sizeof(char *) * numkeys); -+ key_seen = x_malloc(sizeof(bool) * numkeys); -+ key_lengths = x_malloc(sizeof(size_t) * numkeys); -+ value_offsets = x_malloc(sizeof(size_t) * numkeys); -+ value_lengths = x_malloc(sizeof(int) * numkeys); -+ -+ buflen = 0; -+ for (i = 0; i < numkeys; i++) { -+ n = ntohl(*((uint32_t *) (p + 16))); -+ keys[i] = format_hash_as_string((const unsigned char *) p, n); -+ key_lengths[i] = strlen(keys[i]); -+ key_seen[i] = false; -+ cc_log("memcached_mget %.*s %d", (int) key_lengths[i], keys[i], n); -+ value_offsets[i] = buflen; -+ value_lengths[i] = n; -+ buflen += n; -+ p += 20; -+ } -+ assert(buflen == totalsize); -+ -+ buf = x_malloc(buflen); -+ -+ ret = memcached_mget(ptr, (const char *const *) keys, key_lengths, numkeys); -+ if (ret) { -+ cc_log("Failed to mget keys in memcached: %s", -+ memcached_strerror(memc, ret)); -+ for (i = 0; i < numkeys; i++) { -+ free(keys[i]); -+ } -+ free(keys); -+ free(key_lengths); -+ return NULL; -+ } -+ -+ result = NULL; -+ do { -+ const char *k; -+ size_t l; -+ -+ result = memcached_fetch_result(ptr, result, &ret); -+ if (ret == MEMCACHED_END) { -+ break; -+ } -+ if (ret) { -+ cc_log("Failed to get key in memcached: %s", -+ memcached_strerror(memc, ret)); -+ return NULL; -+ } -+ k = memcached_result_key_value(result); -+ l = memcached_result_key_length(result); -+ p = NULL; -+ for (i = 0; i < numkeys; i++) { -+ if (l != key_lengths[i]) { -+ continue; -+ } -+ if (memcmp(k, keys[i], l) == 0) { -+ p = buf + value_offsets[i]; -+ break; -+ } -+ } -+ if (!p) { -+ cc_log("Unknown key was returned: %s", k); -+ return NULL; -+ } -+ if (key_seen[i]) { -+ cc_log("Have already seen chunk: %s", k); -+ return NULL; -+ } -+ key_seen[i] = true; -+ n = memcached_result_length(result); -+ v = memcached_result_value(result); -+ if (n != value_lengths[i]) { -+ cc_log("Unexpected length was returned"); -+ return NULL; -+ } -+ memcpy(p, v, n); -+ } while (ret == MEMCACHED_SUCCESS); -+ -+ for (i = 0; i < numkeys; i++) { -+ if (!key_seen[i]) { -+ cc_log("Failed to get all %d chunks", numkeys); -+ return NULL; -+ } -+ } -+ cc_log("memcached_get %.*s %zu (%zu)", (int) key_length, key, *value_length, -+ buflen); -+ for (i = 0; i < numkeys; i++) { -+ free(keys[i]); -+ } -+ free(keys); -+ free(key_lengths); -+ free(value_offsets); -+ free(value_lengths); -+ -+ *value_length = buflen; -+ return buf; -+} -+ -+int memccached_raw_set(const char *key, const char *data, size_t len) -+{ -+ memcached_return_t mret; -+ -+ mret = memcached_set(memc, key, strlen(key), data, len, 0, 0); -+ if (mret != MEMCACHED_SUCCESS) { -+ cc_log("Failed to move %s to memcached: %s", key, -+ memcached_strerror(memc, mret)); -+ return -1; -+ } -+ return 0; -+} -+ -+/* blob format for storing: -+ -+ char magic[4]; # 'CCH1', might change for other version of ccache -+ # ccache will erase the blob in memcached if wrong magic -+ uint32_t obj_len; # network endian -+ char *obj[obj_len]; -+ uint32_t stderr_len; # network endian -+ char *stderr[stderr_len]; -+ uint32_t dia_len; # network endian -+ char *dia[dia_len]; -+ uint32_t dep_len; # network endian -+ char *dep[dep_len]; -+ -+ */ -+int memccached_set(const char *key, -+ const char *obj, -+ const char *stderr, -+ const char *dia, -+ const char *dep, -+ size_t obj_len, -+ size_t stderr_len, -+ size_t dia_len, -+ size_t dep_len) -+{ -+ size_t buf_len = 4 + 4*4 + obj_len + stderr_len + dia_len + dep_len; -+ char *buf = x_malloc(buf_len); -+ char *ptr; -+ memcached_return_t mret; -+ -+ memcpy(buf, MEMCCACHE_MAGIC, 4); -+ ptr = buf + 4; -+ -+#define PROCESS_ONE_BUFFER(src_ptr, src_len) \ -+ do { \ -+ *((uint32_t *)ptr) = htonl(src_len); \ -+ ptr += 4; \ -+ if (src_len > 0) { \ -+ memcpy(ptr, src_ptr, src_len); \ -+ } \ -+ ptr += src_len; \ -+ } while (false) -+ -+ PROCESS_ONE_BUFFER(obj, obj_len); -+ PROCESS_ONE_BUFFER(stderr, stderr_len); -+ PROCESS_ONE_BUFFER(dia, dia_len); -+ PROCESS_ONE_BUFFER(dep, dep_len); -+ -+#undef PROCESS_ONE_BUFFER -+ -+ if (buf_len > MAX_VALUE_SIZE) { -+ mret = memccached_big_set(memc, key, strlen(key), buf, buf_len, 0, 0); -+ } else { -+ mret = memcached_set(memc, key, strlen(key), buf, buf_len, 0, 0); -+ } -+ -+ if (mret != MEMCACHED_SUCCESS) { -+ cc_log("Failed to move %s to memcached: %s", key, -+ memcached_strerror(memc, mret)); -+ return -1; -+ } -+ return 0; -+} -+ -+static void *memccached_prune(const char *key) -+{ -+ cc_log("key from memcached has wrong data %s: pruning...", key); -+ /* don't really care whether delete failed */ -+ memcached_delete(memc, key, strlen(key), 0); -+ return NULL; -+} -+ -+void *memccached_raw_get(const char *key, char **data, size_t *size) -+{ -+ memcached_return_t mret; -+ void *value; -+ size_t value_l; -+ -+ value = memcached_get(memc, key, strlen(key), &value_l, -+ NULL /*flags*/, &mret); -+ if (!value) { -+ cc_log("Failed to get key from memcached %s: %s", key, -+ memcached_strerror(memc, mret)); -+ return NULL; -+ } -+ *data = value; -+ *size = value_l; -+ return value; /* caller must free this when done with the ptr */ -+} -+ -+void *memccached_get(const char *key, -+ char **obj, -+ char **stderr, -+ char **dia, -+ char **dep, -+ size_t *obj_len, -+ size_t *stderr_len, -+ size_t *dia_len, -+ size_t *dep_len) -+{ -+ memcached_return_t mret; -+ char *value, *ptr; -+ size_t value_l; -+ value = memcached_get(memc, key, strlen(key), &value_l, -+ NULL /*flags*/, &mret); -+ if (!value) { -+ cc_log("Failed to get key from memcached %s: %s", key, -+ memcached_strerror(memc, mret)); -+ return NULL; -+ } -+ if (value_l > 4 && memcmp(value, MEMCCACHE_BIG, 4) == 0) { -+ value = memccached_big_get(memc, key, strlen(key), value, &value_l, -+ NULL /*flags*/, &mret); -+ } -+ if (!value) { -+ cc_log("Failed to get key from memcached %s: %s", key, -+ memcached_strerror(memc, mret)); -+ return NULL; -+ } -+ if (value_l < 20 || memcmp(value, MEMCCACHE_MAGIC, 4) != 0) { -+ cc_log("wrong magic or length %.4s: %d", value, (int)value_l); -+ free(value); -+ return memccached_prune(key); -+ } -+ ptr = value; -+ /* skip the magic */ -+ ptr += 4; -+ value_l -= 4; -+ -+#define PROCESS_ONE_BUFFER(dst_ptr, dst_len) \ -+ do { \ -+ if (value_l < 4) { \ -+ free(value); \ -+ cc_log("no more buffer for %s: %d", \ -+ #dst_ptr, (int)value_l); \ -+ return memccached_prune(key); \ -+ } \ -+ dst_len = ntohl(*((uint32_t *)ptr)); \ -+ ptr += 4; value_l -= 4; \ -+ if (value_l < dst_len) { \ -+ cc_log("no more buffer for %s: %d %d", \ -+ #dst_ptr, (int)value_l, (int) dst_len); \ -+ free(value); \ -+ return memccached_prune(key); \ -+ } \ -+ dst_ptr = ptr; \ -+ ptr += dst_len; value_l -= dst_len; \ -+ } while (false) -+ -+ PROCESS_ONE_BUFFER(*obj, *obj_len); -+ PROCESS_ONE_BUFFER(*stderr, *stderr_len); -+ PROCESS_ONE_BUFFER(*dia, *dia_len); -+ PROCESS_ONE_BUFFER(*dep, *dep_len); -+ -+#undef PROCESS_ONE_BUFFER -+ -+ return value; /* caller must free this when done with the ptrs */ -+} -+ -+void memccached_free(void *blob) -+{ -+ free(blob); -+} -+ -+int memccached_release(void) -+{ -+ memcached_free(memc); -+ return 1; -+} -+ -+#endif /* HAVE_LIBMEMCACHED */ -diff --git a/test.sh b/test.sh -index 3e04157..ac3eb6d 100755 ---- a/test.sh -+++ b/test.sh -@@ -43,6 +43,11 @@ test_failed() { - $CCACHE -s - echo - echo "Test data and log file have been left in $TESTDIR" -+ tail -n 50 $CCACHE_LOGFILE -+ if [ ! -z $CCACHE_MEMCACHED_CONF ]; then -+ memstat --servers=localhost:22122 -+ kill %1 -+ fi - exit 1 - } - -@@ -244,13 +251,13 @@ base_tests() { - $CCACHE_COMPILE -c test1.c - expect_stat 'cache hit (preprocessed)' 0 - expect_stat 'cache miss' 1 -- expect_stat 'files in cache' 1 -+ $CCACHE_NOFILES expect_stat 'files in cache' 1 - expect_equal_object_files reference_test1.o test1.o - - $CCACHE_COMPILE -c test1.c - expect_stat 'cache hit (preprocessed)' 1 - expect_stat 'cache miss' 1 -- expect_stat 'files in cache' 1 -+ $CCACHE_NOFILES expect_stat 'files in cache' 1 - expect_equal_object_files reference_test1.o test1.o - - # ------------------------------------------------------------------------- -@@ -259,7 +266,7 @@ base_tests() { - $CCACHE_COMPILE -c test1.c -g - expect_stat 'cache hit (preprocessed)' 0 - expect_stat 'cache miss' 1 -- expect_stat 'files in cache' 1 -+ $CCACHE_NOFILES expect_stat 'files in cache' 1 - - $CCACHE_COMPILE -c test1.c -g - expect_stat 'cache hit (preprocessed)' 1 -@@ -602,7 +609,7 @@ b" - done - expect_stat 'cache hit (preprocessed)' 0 - expect_stat 'cache miss' 32 -- expect_stat 'files in cache' 32 -+ $CCACHE_NOFILES expect_stat 'files in cache' 32 - - # ------------------------------------------------------------------------- - TEST "Called for preprocessing" -@@ -1366,6 +1373,52 @@ SUITE_masquerading() { - - # ============================================================================= - -+SUITE_memcached_SETUP() { -+ generate_code 1 test1.c -+} -+ -+SUITE_memcached() { -+ export CCACHE_MEMCACHED_CONF=--SERVER=localhost:22122 -+ memcached -p 22122 & -+ memcached_pid=$! -+ base_tests -+ kill $memcached_pid -+ unset CCACHE_MEMCACHED_CONF -+} -+ -+SUITE_memcached_only_SETUP() { -+ generate_code 1 test1.c -+} -+ -+SUITE_memcached_only() { -+ CCACHE_NOFILES=true -+ export CCACHE_MEMCACHED_CONF=--SERVER=localhost:22122 -+ export CCACHE_MEMCACHED_ONLY=1 -+ memcached -p 22122 & -+ memcached_pid=$! -+ base_tests -+ kill $memcached_pid -+ unset CCACHE_MEMCACHED_CONF -+ unset CCACHE_MEMCACHED_ONLY -+ unset CCACHE_NOFILES -+} -+ -+SUITE_memcached_socket_SETUP() { -+ generate_code 1 test1.c -+} -+ -+SUITE_memcached_socket() { -+ export CCACHE_MEMCACHED_CONF=--SOCKET=\"/tmp/memcached.$$\" -+ memcached -s /tmp/memcached.$$ & -+ memcached_pid=$! -+ base_tests -+ kill $memcached_pid -+ rm /tmp/memcached.$$ -+ unset CCACHE_MEMCACHED_CONF -+} -+ -+# ============================================================================= -+ - SUITE_hardlink_PROBE() { - touch file1 - if ! ln file1 file2 >/dev/null 2>&1; then -@@ -1539,7 +1592,7 @@ EOF - test_failed "$dep_file missing" - fi - done -- expect_stat 'files in cache' 12 -+ $CCACHE_NOFILES expect_stat 'files in cache' 12 - - # ------------------------------------------------------------------------- - TEST "-Wp,-MD" -@@ -3412,6 +3465,14 @@ upgrade - input_charset - " - -+if [ ! -z $CCACHE_MEMCACHED ]; then -+ all_suites="$all_suites -+memcached -+memcached_only -+memcached_socket -+" -+fi -+ - compiler_location=$(which $(echo "$COMPILER" | awk '{print $1}')) - if [ "$compiler_location" = "$COMPILER" ]; then - echo "Compiler: $COMPILER" -diff --git a/test/test_conf.c b/test/test_conf.c -index ea43e2e..d65372c 100644 ---- a/test/test_conf.c -+++ b/test/test_conf.c -@@ -18,7 +18,7 @@ - #include "framework.h" - #include "util.h" - --#define N_CONFIG_ITEMS 31 -+#define N_CONFIG_ITEMS 34 - static struct { - char *descr; - const char *origin; -@@ -68,11 +68,14 @@ TEST(conf_create) - CHECK_STR_EQ("", conf->log_file); - CHECK_INT_EQ(0, conf->max_files); - CHECK_INT_EQ((uint64_t)5 * 1000 * 1000 * 1000, conf->max_size); -+ CHECK_STR_EQ("", conf->memcached_conf); -+ CHECK(!conf->memcached_only); - CHECK_STR_EQ("", conf->path); - CHECK_STR_EQ("", conf->prefix_command); - CHECK_STR_EQ("", conf->prefix_command_cpp); - CHECK(!conf->read_only); - CHECK(!conf->read_only_direct); -+ CHECK(!conf->read_only_memcached); - CHECK(!conf->recache); - CHECK(conf->run_second_cpp); - CHECK_INT_EQ(0, conf->sloppiness); -@@ -119,11 +122,14 @@ TEST(conf_read_valid_config) - "log_file = $USER${USER} \n" - "max_files = 17\n" - "max_size = 123M\n" -+ "memcached_conf = --SERVER=localhost\n" -+ "memcached_only = true\n" - "path = $USER.x\n" - "prefix_command = x$USER\n" - "prefix_command_cpp = y\n" - "read_only = true\n" - "read_only_direct = true\n" -+ "read_only_memcached = false\n" - "recache = true\n" - "run_second_cpp = false\n" - "sloppiness = file_macro ,time_macros, include_file_mtime,include_file_ctime,file_stat_matches,pch_defines , no_system_headers \n" -@@ -157,11 +163,14 @@ TEST(conf_read_valid_config) - CHECK_STR_EQ_FREE1(format("%s%s", user, user), conf->log_file); - CHECK_INT_EQ(17, conf->max_files); - CHECK_INT_EQ(123 * 1000 * 1000, conf->max_size); -+ CHECK_STR_EQ("--SERVER=localhost", conf->memcached_conf); -+ CHECK(conf->memcached_only); - CHECK_STR_EQ_FREE1(format("%s.x", user), conf->path); - CHECK_STR_EQ_FREE1(format("x%s", user), conf->prefix_command); - CHECK_STR_EQ("y", conf->prefix_command_cpp); - CHECK(conf->read_only); - CHECK(conf->read_only_direct); -+ CHECK(!conf->read_only_memcached); - CHECK(conf->recache); - CHECK(!conf->run_second_cpp); - CHECK_INT_EQ(SLOPPY_INCLUDE_FILE_MTIME|SLOPPY_INCLUDE_FILE_CTIME| -@@ -383,11 +392,14 @@ TEST(conf_print_items) - "lf", - 4711, - 98.7 * 1000 * 1000, -+ "mc", -+ false, - "p", - "pc", - "pcc", - true, - true, -+ false, - true, - .run_second_cpp = false, - SLOPPY_FILE_MACRO|SLOPPY_INCLUDE_FILE_MTIME| -@@ -433,11 +445,14 @@ TEST(conf_print_items) - CHECK_STR_EQ("log_file = lf", received_conf_items[n++].descr); - CHECK_STR_EQ("max_files = 4711", received_conf_items[n++].descr); - CHECK_STR_EQ("max_size = 98.7M", received_conf_items[n++].descr); -+ CHECK_STR_EQ("memcached_conf = mc", received_conf_items[n++].descr); -+ CHECK_STR_EQ("memcached_only = false", received_conf_items[n++].descr); - CHECK_STR_EQ("path = p", received_conf_items[n++].descr); - CHECK_STR_EQ("prefix_command = pc", received_conf_items[n++].descr); - CHECK_STR_EQ("prefix_command_cpp = pcc", received_conf_items[n++].descr); - CHECK_STR_EQ("read_only = true", received_conf_items[n++].descr); - CHECK_STR_EQ("read_only_direct = true", received_conf_items[n++].descr); -+ CHECK_STR_EQ("read_only_memcached = false", received_conf_items[n++].descr); - CHECK_STR_EQ("recache = true", received_conf_items[n++].descr); - CHECK_STR_EQ("run_second_cpp = false", received_conf_items[n++].descr); - CHECK_STR_EQ("sloppiness = file_macro, include_file_mtime," -diff --git a/upload-memcached.py b/upload-memcached.py -new file mode 100755 -index 0000000..bc489b0 ---- /dev/null -+++ b/upload-memcached.py -@@ -0,0 +1,126 @@ -+#!/usr/bin/env python -+ -+import memcache -+import struct -+import os -+import hashlib -+ -+""" -+/* blob format for storing: -+ -+ char magic[4]; # 'CCH1', might change for other version of ccache -+ # ccache will erase the blob in memcached if wrong magic -+ uint32_t obj_len; # network endian -+ char *obj[obj_len]; -+ uint32_t stderr_len; # network endian -+ char *stderr[stderr_len]; -+ uint32_t dia_len; # network endian -+ char *dia[dia_len]; -+ uint32_t dep_len; # network endian -+ char *dep[dep_len]; -+ -+*/ -+""" -+MEMCCACHE_MAGIC = 'CCH1' -+ -+def set_blob(data): -+ return struct.pack('!I', len(data)) + str(data) -+MEMCCACHE_BIG = 'CCBM' -+ -+""" -+/* blob format for big values: -+ -+ char magic[4]; # 'CCBM' -+ uint32_t numkeys; # network endian -+ uint32_t hash_size; # network endian -+ uint32_t reserved; # network endian -+ uint32_t value_length; # network endian -+ -+ hash of include file ( bytes) -+ size of include file (4 bytes unsigned int) -+ ... -+ -+ -+ -+*/ -+""" -+MEMCCACHE_BIG = 'CCBM' -+ -+MAX_VALUE_SIZE = 1000 << 10 # 1M with memcached overhead -+SPLIT_VALUE_SIZE = MAX_VALUE_SIZE -+ -+server = os.getenv("MEMCACHED_SERVERS", "localhost") -+mc = memcache.Client(server.split(','), debug=1) -+ -+ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.ccache")) -+filelist = [] -+for dirpath, dirnames, filenames in os.walk(ccache): -+ # sort by modification time, most recently used last -+ for filename in filenames: -+ stat = os.stat(os.path.join(dirpath, filename)) -+ filelist.append((stat.st_mtime, dirpath, filename)) -+filelist.sort() -+files = blobs = chunks = objects = manifest = 0 -+for mtime, dirpath, filename in filelist: -+ dirname = dirpath.replace(ccache + os.path.sep, "") -+ if filename == "CACHEDIR.TAG": -+ # ignore these -+ files = files + 1 -+ else: -+ (base, ext) = os.path.splitext(filename) -+ if ext == '.o': -+ objects = objects + 1 -+ key = "".join(list(os.path.split(dirname)) + [base]) -+ def read_file(path): -+ return os.path.exists(path) and open(path).read() or "" -+ obj = read_file(os.path.join(dirpath, filename)) -+ stderr = read_file(os.path.join(dirpath, base) + '.stderr') -+ dia = read_file(os.path.join(dirpath, base) + '.dia') -+ dep = read_file(os.path.join(dirpath, base) + '.d') -+ -+ print "%s: %d %d %d %d" % (key, len(obj), len(stderr), len(dia), len(dep)) -+ val = MEMCCACHE_MAGIC -+ val += set_blob(obj) -+ val += set_blob(stderr) -+ val += set_blob(dia) -+ val += set_blob(dep) -+ if len(val) > MAX_VALUE_SIZE: -+ numkeys = (len(val) + SPLIT_VALUE_SIZE - 1) / SPLIT_VALUE_SIZE -+ buf = MEMCCACHE_BIG -+ buf += struct.pack('!I', numkeys) -+ buf += struct.pack('!I', 16) -+ buf += struct.pack('!I', 0) -+ buf += struct.pack('!I', len(val)) -+ def splitchunks(s, n): -+ """Produce `n`-character chunks from `s`.""" -+ for start in range(0, len(s), n): -+ yield s[start:start+n] -+ valmap = {} -+ for subval in splitchunks(val, SPLIT_VALUE_SIZE): -+ subhash = hashlib.new('md4') -+ subhash.update(subval) -+ buf += subhash.digest() + struct.pack('!I', len(subval)) -+ subkey = "%s-%d" % (subhash.hexdigest(), len(subval)) -+ print "# %s: chunk %d" % (subkey, len(subval)) -+ #mc.set(subkey, subval) -+ valmap[subkey] = subval -+ chunks = chunks + 1 -+ mc.set_multi(valmap) -+ mc.set(key, buf) -+ else: -+ mc.set(key, val) -+ files = files + 1 -+ blobs = blobs + 1 -+ elif ext == '.stderr' or ext == '.d' or ext == '.dia': -+ # was added above -+ files = files + 1 -+ elif ext == '.manifest': -+ manifest = manifest + 1 -+ key = "".join(list(os.path.split(dirname)) + [base]) -+ val = open(os.path.join(dirpath, filename)).read() or None -+ if val: -+ print "%s: manifest %d" % (key, len(val)) -+ mc.set(key, val, 0, 0) -+ files = files + 1 -+ blobs = blobs + 1 -+print "%d files, %d objects (%d manifest) = %d blobs (%d chunks)" % (files, objects, manifest, blobs, chunks) -diff --git a/util.c b/util.c -index f048d97..6059f25 100644 ---- a/util.c -+++ b/util.c -@@ -388,6 +388,75 @@ copy_file(const char *src, const char *dest, int compress_level) - return -1; - } - -+// Write data to a fd. -+int safe_write(int fd_out, const char *data, size_t length) -+{ -+ size_t written = 0; -+ do { -+ int ret; -+ ret = write(fd_out, data + written, length - written); -+ if (ret < 0) { -+ if (errno != EAGAIN && errno != EINTR) { -+ return ret; -+ } -+ } else { -+ written += ret; -+ } -+ } while (written < length); -+ return 0; -+} -+ -+// Write data to a file. -+int write_file(const char *data, const char *dest, size_t length) -+{ -+ int fd_out; -+ char *tmp_name; -+ int ret; -+ int saved_errno = 0; -+ -+ tmp_name = x_strdup(dest); -+ fd_out = create_tmp_fd(&tmp_name); -+ if (fd_out < 0) { -+ tmp_unlink(tmp_name); -+ free(tmp_name); -+ return -1; -+ } -+ -+ ret = safe_write(fd_out, data, length); -+ if (ret < 0) { -+ saved_errno = errno; -+ cc_log("write error: %s", strerror(saved_errno)); -+ goto error; -+ } -+ -+#ifndef _WIN32 -+ fchmod(fd_out, 0666 & ~get_umask()); -+#endif -+ -+ /* the close can fail on NFS if out of space */ -+ if (close(fd_out) == -1) { -+ saved_errno = errno; -+ cc_log("close error: %s", strerror(saved_errno)); -+ goto error; -+ } -+ -+ if (x_rename(tmp_name, dest) == -1) { -+ saved_errno = errno; -+ cc_log("rename error: %s", strerror(saved_errno)); -+ goto error; -+ } -+ -+ free(tmp_name); -+ return 0; -+ -+error: -+ close(fd_out); -+ tmp_unlink(tmp_name); -+ free(tmp_name); -+ errno = saved_errno; -+ return -1; -+} -+ - // Run copy_file() and, if successful, delete the source file. - int - move_file(const char *src, const char *dest, int compress_level) Property changes on: head/devel/ccache/files/extra-patch-memcached ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -yes \ No newline at end of property Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/devel/ccache/files/extra-patch-memcached-configure.ac =================================================================== --- head/devel/ccache/files/extra-patch-memcached-configure.ac (revision 473409) +++ head/devel/ccache/files/extra-patch-memcached-configure.ac (revision 473410) @@ -1,59 +1,59 @@ --- configure.ac.orig 2017-07-27 13:36:38.827581000 -0700 +++ configure.ac 2017-07-27 13:39:10.856958000 -0700 @@ -28,6 +28,10 @@ AC_PROG_CC_C99 if test "$ac_cv_prog_cc_c99" = no; then AC_MSG_ERROR(cannot find a C99-compatible compiler) fi +AC_PROG_CXX +if test "$ac_cv_prog_cxx" = no; then + AC_MSG_ERROR(cannot find a C++-compatible compiler) +fi AC_PROG_CPP AC_PROG_INSTALL @@ -97,19 +101,39 @@ AC_ARG_ENABLE(memcached, [AS_HELP_STRING([--enable-memcached], [enable memcached as a cache backend])]) +# Need to use C++ compiler for linking -lmemcached as static, rather than +# hacking in GCC's -lstdc++, since we may be using Clang. +AC_LANG(C++) dnl enable-memcached: Check if -lmemcached is needed. if test x${enable_memcached} != x; then - if test x${enable_static} != x; then + if false && test x${enable_static} != x; then AC_CHECK_LIB(stdc++, __gxx_personality_v0,[]) fi AC_CHECK_LIB(pthread, pthread_once) - AC_CHECK_LIB(memcached, memcached,[],[ - echo ' WARNING: recent version libmemcached not found' - echo ' please install libmemcached > 1.0 with development files' - exit 1 - ]) -+ if test x${enable_static} != x; then ++ if test x${enable_static} = xyes; then + AC_CHECK_LIB(sasl2, sasl_version, ac_cv_have_libsasl2=yes, + ac_cv_have_libsasl2=no, -lcrypto -lopie -lmd) + fi + if test x$ac_cv_have_libsasl2 = xyes; then + AC_CHECK_LIB(memcached, memcached,[ + LIBS="${LIBS} -lmemcached -lsasl2 -lcrypto -lopie -lmd" + AC_DEFINE(HAVE_LIBMEMCACHED, 1) + ] + ,[ + echo ' WARNING: recent version libmemcached not found' + echo ' please install libmemcached > 1.0 with development files' + exit 1 + ], -lsasl2 -lcrypto -lopie -lmd) + else + AC_CHECK_LIB(memcached, memcached,[],[ + echo ' WARNING: recent version libmemcached not found' + echo ' please install libmemcached > 1.0 with development files' + exit 1 + ]) + fi ccache_memcached='CCACHE_MEMCACHED=1 ' fi +AC_LANG(C) dnl Check for zlib AC_ARG_WITH(bundled-zlib, Index: head/devel/ccache/files/patch-src__ccache.c =================================================================== --- head/devel/ccache/files/patch-src__ccache.c (nonexistent) +++ head/devel/ccache/files/patch-src__ccache.c (revision 473410) @@ -0,0 +1,56 @@ +- Determine whether cc(1) is clang or gcc at compile-time. +- Don't hash -fcolor-diagnostics; make will auto use it while make -j will not. + There's no reason to not use the cache in either of these cases if it is + already available. +-bdrewery + + +--- src/ccache.c.orig 2018-03-25 13:24:05.000000000 -0700 ++++ src/ccache.c 2018-06-26 11:44:03.336177000 -0700 +@@ -487,6 +487,16 @@ guess_compiler(const char *path) + } else if (str_eq(name, "pump") || str_eq(name, "distcc-pump")) { + result = GUESSED_PUMP; + } ++#if defined(CC_IS_CLANG) || defined(CC_IS_GCC) ++ if (result == GUESSED_UNKNOWN && ++ (strcmp(name, "cc") == 0 || strcmp(name, "CC") == 0 || ++ strcmp(name, "c++") == 0)) ++#if defined(CC_IS_CLANG) ++ result = GUESSED_CLANG; ++#elif defined(CC_IS_GCC) ++ result = GUESSED_GCC; ++#endif ++#endif + free(name); + return result; + } +@@ -1657,6 +1667,7 @@ calculate_common_hash(struct args *args, struct mdfour + free(p); + } + ++#if 0 + // Possibly hash GCC_COLORS (for color diagnostics). + if (guessed_compiler == GUESSED_GCC) { + const char *gcc_colors = getenv("GCC_COLORS"); +@@ -1665,6 +1676,7 @@ calculate_common_hash(struct args *args, struct mdfour + hash_string(hash, gcc_colors); + } + } ++#endif + } + + // Update a hash sum with information specific to the direct and preprocessor +@@ -1697,6 +1709,13 @@ calculate_object_hash(struct args *args, struct mdfour + + // -Wl,... doesn't affect compilation (except for clang). + if (str_startswith(args->argv[i], "-Wl,") && !is_clang) { ++ continue; ++ } ++ ++ /* Colors do not affect compilation. */ ++ if (str_startswith(args->argv[i], "-fcolor-diagnostics") || ++ str_eq(args->argv[i], "-fdiagnostics-color") || ++ str_eq(args->argv[i], "-fdiagnostics-color=always")) { + continue; + } + Property changes on: head/devel/ccache/files/patch-src__ccache.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +yes \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property