Index: head/devel/ccache/Makefile =================================================================== --- head/devel/ccache/Makefile (revision 459886) +++ head/devel/ccache/Makefile (revision 459887) @@ -1,126 +1,126 @@ # Created by: Dominic Marks # $FreeBSD$ PORTNAME= ccache -PORTVERSION= 3.3.4 -PORTREVISION= 8 +PORTVERSION= 3.3.5 +PORTREVISION= 0 CATEGORIES= devel MASTER_SITES= http://www.samba.org/ftp/ccache/ \ LOCAL/bdrewery MAINTAINER= bdrewery@FreeBSD.org COMMENT= Tool to minimize the compile time of C/C++ programs LICENSE= GPLv3+ LICENSE_FILE= ${WRKSRC}/GPL-3.0.txt CONFLICTS_INSTALL= ccache-static-[0-9]* ccache-memcached-[0-9]* ccache-memcached-static-[0-9]* GNU_CONFIGURE= yes HOWTO= ccache-howto-freebsd.txt CCLINKDIR= libexec/ccache SUB_FILES= ${HOWTO} world-ccache pkg-message ccache-update-links.sh PORTDOCS= ccache-howto-freebsd.txt MANUAL.html MANUAL.txt OPTIONS_DEFINE= CLANGLINK LLVMLINK STATIC DOCS TINDERBOX MEMCACHED OPTIONS_DEFAULT=CLANGLINK LLVMLINK CLANGLINK_DESC= Create clang compiler links if clang is installed LLVMLINK_DESC= Create llvm compiler links if llvm is installed TINDERBOX_DESC= Create tarball for tinderbox usage MEMCACHED_DESC= Build in experimental Memcached support USES= compiler MEMCACHED_EXTRA_PATCHES= ${FILESDIR}/extra-patch-memcached:-p1 \ ${FILESDIR}/extra-patch-memcached-ccache.c \ ${FILESDIR}/extra-patch-memcached-configure.ac \ ${FILESDIR}/extra-patch-memcached-Makefile.in MEMCACHED_CONFIGURE_ENABLE= memcached MEMCACHED_USES= autoreconf pkgconfig MEMCACHED_LIB_DEPENDS= libmemcached.so:databases/libmemcached MEMCACHED_LDFLAGS= -L${LOCALBASE}/lib MEMCACHED_CFLAGS= -I${LOCALBASE}/include STATIC_CONFIGURE_ENABLE= static .if defined(WITH_CCACHE_BUILD) && empty(OPTIONS_SLAVE:MMEMCACHED) && \ !defined(NO_CCACHE_DEPEND) # Don't allow autoreconf. We want no dependencies on this to keep # WITH_CCACHE_BUILD working. USES:= ${USES:Nautoreconf} MEMCACHED_IGNORE= MEMCACHED cannot be combined with WITH_CCACHE_BUILD. Use devel/ccache-memcached # XXX: This needs more testing with Poudriere before enabling. Also bsd.options.mk support. #MEMCACHED_DEPENDS_ARGS+= NO_CCACHE_DEPEND=1 .endif # Support WITH_CCACHE_BUILD but don't depend on it. NO_CCACHE_DEPEND= yes OPTIONS_SUB= yes STATIC_LDFLAGS= -static .include . if ${COMPILER_TYPE} == clang CPPFLAGS+= -DCC_IS_CLANG . elif ${COMPILER_TYPE} == gcc CPPFLAGS+= -DCC_IS_GCC . endif PLIST_SUB+= CCLINKDIR="${CCLINKDIR}" .if ${ARCH}=="i386" CCACHE_COMPILERS+= icc icpc .endif GNU_COMPILERS+= 34 42 43 44 45 46 47 48 49 5 6 7 8 CCACHE_COMPILERS+= cc c++ CC gcc g++ ${GNU_COMPILERS:S|^|gcc|} ${GNU_COMPILERS:S|^|g++|} .if ${PORT_OPTIONS:MCLANGLINK} CLANG_COMPILERS+= 33 34 35 36 37 38 39 40 50 -devel CCACHE_COMPILERS+= clang clang++ ${CLANG_COMPILERS:S|^|clang|} ${CLANG_COMPILERS:S|^|clang++|} .endif .if ${PORT_OPTIONS:MLLVMLINK} CCACHE_COMPILERS+= llvm-gcc llvm-c++ llvm-g++ .endif CCACHE_COMPILERS+= ${EXTRA_COMPILERS} SUB_LIST+= CCACHE_COMPILERS="${CCACHE_COMPILERS}" \ CCLINKDIR="${CCLINKDIR}" \ ICCPREFIX="${LOCALBASE}/intel_cc_80/bin" \ HOWTO="${HOWTO}" post-build-TINDERBOX-on: @${MKDIR} ${WRKDIR}/tb/opt @${INSTALL_PROGRAM} ${WRKSRC}/${PORTNAME} ${WRKDIR}/tb/opt .for l in ${CCACHE_COMPILERS} @${LN} -sf ${PORTNAME} ${WRKDIR}/tb/opt/${l} .endfor @${TAR} -C ${WRKDIR}/tb -cpf ${WRKSRC}/${PORTNAME}.tar opt do-install: ${INSTALL_PROGRAM} ${WRKSRC}/ccache ${STAGEDIR}${PREFIX}/bin ${INSTALL_MAN} ${WRKSRC}/ccache.1 ${STAGEDIR}${PREFIX}/man/man1 ${MKDIR} ${STAGEDIR}${PREFIX}/${CCLINKDIR}/world ${INSTALL_SCRIPT} ${WRKDIR}/world-ccache \ ${STAGEDIR}${PREFIX}/${CCLINKDIR}/world/ccache ${INSTALL_SCRIPT} ${WRKDIR}/ccache-update-links.sh \ ${STAGEDIR}${PREFIX}/bin/ccache-update-links do-install-TINDERBOX-on: ${MKDIR} ${STAGEDIR}${DATADIR} ${INSTALL_DATA} ${WRKSRC}/${PORTNAME}.tar ${STAGEDIR}${DATADIR} do-install-DOCS-on: ${MKDIR} ${STAGEDIR}${DOCSDIR} ${INSTALL_DATA} ${WRKSRC}/MANUAL.html ${STAGEDIR}${DOCSDIR} ${INSTALL_DATA} ${WRKSRC}/MANUAL.txt ${STAGEDIR}${DOCSDIR} ${INSTALL_DATA} ${WRKDIR}/${HOWTO} ${STAGEDIR}${DOCSDIR} .include Index: head/devel/ccache/distinfo =================================================================== --- head/devel/ccache/distinfo (revision 459886) +++ head/devel/ccache/distinfo (revision 459887) @@ -1,3 +1,3 @@ -TIMESTAMP = 1487435289 -SHA256 (ccache-3.3.4.tar.gz) = 1348b54e7c35dd2f8d17923389e03c546e599cfbde6459d2f31cf6f1521ec538 -SIZE (ccache-3.3.4.tar.gz) = 449905 +TIMESTAMP = 1516823791 +SHA256 (ccache-3.3.5.tar.gz) = c84b2460dab2bd9b8b743499a51910c6b7149bc0271d0f6000e0ad1f6c8fda2b +SIZE (ccache-3.3.5.tar.gz) = 453896 Index: head/devel/ccache/files/extra-patch-memcached =================================================================== --- head/devel/ccache/files/extra-patch-memcached (revision 459886) +++ head/devel/ccache/files/extra-patch-memcached (revision 459887) @@ -1,2396 +1,2387 @@ https://github.com/ccache/ccache/pull/58 Retrieved on February 13th 2017. Changes to .travis.yml removed since it is not in the release image. diff --git a/MANUAL.txt b/MANUAL.txt index ab01886..c78bb6e 100644 --- a/MANUAL.txt +++ b/MANUAL.txt @@ -418,6 +418,20 @@ WRAPPERS>>. The default value is 5G. Available suffixes: k, M, G, T (decimal) and Ki, Mi, Gi, Ti (binary). The default suffix is "G". +*memcached_conf* (*CCACHE_MEMCACHED_CONF*):: + + The memcached_conf option sets the memcached(3) configuration to use for + storing and getting cache values, if any. Example configuration: ++ +------------------------------------------------------------------------------- +CCACHE_MEMCACHED_CONF=--SERVER=localhost:11211 +------------------------------------------------------------------------------- + +*memcached_only* (*CCACHE_MEMCACHED_ONLY*):: + + Only store files in memcached, don't store them in the local filesystems. + The manifests (for direct mode) and stats are still being stored locally. + *path* (*CCACHE_PATH*):: If set, ccache will search directories in this list when looking for the @@ -451,6 +465,11 @@ WRAPPERS>>. from the cache using the direct mode, not the preprocessor mode. See documentation for *read_only* regarding using a read-only ccache directory. +*read_only_memcached* (*CCACHE_READONLY_MEMCACHED* or *CCACHE_NOREADONLY_MEMCACHED*), see <<_boolean_values,Boolean values>> above):: + + If true, ccache will attempt to get previously cached values from memcached, + but will not try to store any new values in memcached. + *recache* (*CCACHE_RECACHE* or *CCACHE_NORECACHE*, see <<_boolean_values,Boolean values>> above):: If true, ccache will not use any previously stored result. New results will @@ -769,6 +788,29 @@ A tip is to set *temporary_dir* to a directory on the local host to avoid NFS traffic for temporary files. +Sharing a cache with memcached +------------------------------ + +When using the *memcached* () feature, the most recently +used cache entries are also available from the configured memcached servers. + +The local cache directory will be searched first, but then it will still be +possible to get cache hits (over the network) before having to run the +compiler. + +Using a local *moxi* (memcached proxy) will enable multiple ccache invocations +to share memcached connections and thus avoid some of the network overhead. + +It will also allow you to fine-tune connection timeouts and other settings. You +can optionally replace your memcached servers with Couchbase servers. + +Example: + +------------------------------------------------------------------------------- +moxi -z 11211=mc_server1:11211,mc_server2:11211 +------------------------------------------------------------------------------- + + Using ccache with other compiler wrappers ----------------------------------------- diff --git a/Makefile.in b/Makefile.in index 5aee02d..08b3633 100644 --- a/Makefile.in +++ b/Makefile.in @@ -37,6 +37,7 @@ non_3pp_sources = \ lockfile.c \ manifest.c \ mdfour.c \ + memccached.c \ stats.c \ unify.c \ util.c \ @@ -101,7 +102,7 @@ perf: ccache$(EXEEXT) .PHONY: test test: ccache$(EXEEXT) test/main$(EXEEXT) test/main$(EXEEXT) - CC='$(CC)' $(srcdir)/test.sh + CC='$(CC)' @ccache_memcached@$(srcdir)/test.sh .PHONY: quicktest quicktest: test/main$(EXEEXT) diff --git a/ccache.c b/ccache.c index 88e0ec5..12026c7 100644 --- a/ccache.c +++ b/ccache.c @@ -102,6 +102,9 @@ static char *output_dia = NULL; // Split dwarf information (GCC 4.8 andup). Contains pathname if not NULL. static char *output_dwo = NULL; +// The cached key. +static char *cached_key; + // Array for storing -arch options. #define MAX_ARCH_ARGS 10 static size_t arch_args_size = 0; @@ -123,6 +126,9 @@ static char *cached_stderr; // (cachedir/a/b/cdef[...]-size.d). static char *cached_dep; +// The manifest key. +static char *manifest_name; + // Full path to the file containing the coverage information // (cachedir/a/b/cdef[...]-size.gcno). static char *cached_cov; @@ -239,6 +245,18 @@ static pid_t compiler_pid = 0; // stored in the cache changes in a backwards-incompatible way. static const char HASH_PREFIX[] = "3"; +static void from_fscache(enum fromcache_call_mode mode, + bool put_object_in_manifest); +static void to_fscache(struct args *args); +#ifdef HAVE_LIBMEMCACHED +static void from_memcached(enum fromcache_call_mode mode, + bool put_object_in_manifest); +static void to_memcached(struct args *args); +#endif +static void (*from_cache)(enum fromcache_call_mode mode, + bool put_object_in_manifest); +static void (*to_cache)(struct args *args); + static void add_prefix(struct args *args, char *prefix_command) { @@ -952,6 +970,28 @@ put_file_in_cache(const char *source, const char *dest) stats_update_size(file_size(&st), 1); } +#ifdef HAVE_LIBMEMCACHED +// Copy data to the cache. +static void +put_data_in_cache(void *data, size_t size, const char *dest) +{ + int ret; + + assert(!conf->read_only); + assert(!conf->read_only_direct); + + /* already compressed (in cache) */ + ret = write_file(data, dest, size); + if (ret != 0) { + cc_log("Failed to write to %s: %s", dest, strerror(errno)); + stats_update(STATS_ERROR); + failed(); + } + cc_log("Stored in cache: %zu bytes -> %s", size, dest); + stats_update_size(size, 1); +} +#endif + // Copy or link a file from the cache. static void get_file_from_cache(const char *source, const char *dest) @@ -1006,6 +1046,11 @@ send_cached_stderr(void) // Create or update the manifest file. void update_manifest_file(void) { +#ifdef HAVE_LIBMEMCACHED + char *data; + size_t size; +#endif + if (!conf->direct_mode || !included_files || conf->read_only @@ -1023,6 +1068,14 @@ void update_manifest_file(void) update_mtime(manifest_path); if (x_stat(manifest_path, &st) == 0) { stats_update_size(file_size(&st) - old_size, old_size == 0 ? 1 : 0); +#if HAVE_LIBMEMCACHED + if (strlen(conf->memcached_conf) > 0 && !conf->read_only_memcached && + read_file(manifest_path, st.st_size, &data, &size)) { + cc_log("Storing %s in memcached", manifest_name); + memccached_raw_set(manifest_name, data, size); + free(data); + } +#endif } } else { cc_log("Failed to add object file hash to %s", manifest_path); @@ -1031,8 +1084,12 @@ void update_manifest_file(void) // Run the real compiler and put the result in cache. static void -to_cache(struct args *args) +to_fscache(struct args *args) { +#ifdef HAVE_LIBMEMCACHED + char *data_obj, *data_stderr, *data_dia, *data_dep; + size_t size_obj, size_stderr, size_dia, size_dep; +#endif char *tmp_stdout = format("%s.tmp.stdout", cached_obj); int tmp_stdout_fd = create_tmp_fd(&tmp_stdout); char *tmp_stderr = format("%s.tmp.stderr", cached_obj); @@ -1288,6 +1345,40 @@ to_cache(struct args *args) } } +#ifdef HAVE_LIBMEMCACHED + if (strlen(conf->memcached_conf) > 0 && !conf->read_only_memcached && + !using_split_dwarf && /* no support for the dwo files just yet */ + !generating_coverage) { /* coverage refers to local paths anyway */ + cc_log("Storing %s in memcached", cached_key); + if (!read_file(cached_obj, 0, &data_obj, &size_obj)) { + data_obj = NULL; + size_obj = 0; + } + if (!read_file(cached_stderr, 0, &data_stderr, &size_stderr)) { + data_stderr = NULL; + size_stderr = 0; + } + if (!read_file(cached_dia, 0, &data_dia, &size_dia)) { + data_dia = NULL; + size_dia = 0; + } + if (!read_file(cached_dep, 0, &data_dep, &size_dep)) { + data_dep = NULL; + size_dep = 0; + } + + if (data_obj) { + memccached_set(cached_key, + data_obj, data_stderr, data_dia, data_dep, + size_obj, size_stderr, size_dia, size_dep); + } + + free(data_obj); + free(data_stderr); + free(data_dia); + free(data_dep); + } +#endif // Everything OK. send_cached_stderr(); update_manifest_file(); @@ -1298,6 +1389,226 @@ to_cache(struct args *args) free(tmp_dwo); } +#ifdef HAVE_LIBMEMCACHED +// Run the real compiler and put the result in cache. +static void +to_memcached(struct args *args) +{ + const char *tmp_dir = temp_dir(); + char *tmp_stdout, *tmp_stderr; + char *stderr_d, *obj_d, *dia_d = NULL, *dep_d = NULL; + size_t stderr_l = 0, obj_l = 0, dia_l = 0, dep_l = 0; + struct stat st; + int status, tmp_stdout_fd, tmp_stderr_fd; + + tmp_stdout = format("%s/%s.tmp.stdout.%s", tmp_dir, cached_obj, tmp_string()); + tmp_stdout_fd = create_tmp_fd(&tmp_stdout); + tmp_stderr = format("%s/%s.tmp.stderr.%s", tmp_dir, cached_obj, tmp_string()); + tmp_stderr_fd = create_tmp_fd(&tmp_stderr); + + if (generating_coverage) { + cc_log("No memcached support for coverage yet"); + failed(); + } + if (using_split_dwarf) { + cc_log("No memcached support for split dwarf yet"); + failed(); + } + + if (create_parent_dirs(tmp_stdout) != 0) { + fatal("Failed to create parent directory for %s: %s", + tmp_stdout, strerror(errno)); + } + + args_add(args, "-o"); + args_add(args, output_obj); + + if (output_dia) { + args_add(args, "--serialize-diagnostics"); + args_add(args, output_dia); + } + + /* Turn off DEPENDENCIES_OUTPUT when running cc1, because + * otherwise it will emit a line like + * + * tmp.stdout.vexed.732.o: /home/mbp/.ccache/tmp.stdout.vexed.732.i + */ + x_unsetenv("DEPENDENCIES_OUTPUT"); + + if (conf->run_second_cpp) { + args_add(args, input_file); + } else { + args_add(args, i_tmpfile); + } + + cc_log("Running real compiler"); + status = execute(args->argv, tmp_stdout_fd, tmp_stderr_fd, &compiler_pid); + args_pop(args, 3); + + if (x_stat(tmp_stdout, &st) != 0) { + /* The stdout file was removed - cleanup in progress? Better bail out. */ + stats_update(STATS_MISSING); + tmp_unlink(tmp_stdout); + tmp_unlink(tmp_stderr); + failed(); + } + if (st.st_size != 0) { + cc_log("Compiler produced stdout"); + stats_update(STATS_STDOUT); + tmp_unlink(tmp_stdout); + tmp_unlink(tmp_stderr); + failed(); + } + tmp_unlink(tmp_stdout); + + /* + * Merge stderr from the preprocessor (if any) and stderr from the real + * compiler into tmp_stderr. + */ + if (cpp_stderr) { + int fd_cpp_stderr; + int fd_real_stderr; + int fd_result; + char *tmp_stderr2; + + tmp_stderr2 = format("%s.2", tmp_stderr); + if (x_rename(tmp_stderr, tmp_stderr2)) { + cc_log("Failed to rename %s to %s: %s", tmp_stderr, tmp_stderr2, + strerror(errno)); + failed(); + } + fd_cpp_stderr = open(cpp_stderr, O_RDONLY | O_BINARY); + if (fd_cpp_stderr == -1) { + cc_log("Failed opening %s: %s", cpp_stderr, strerror(errno)); + failed(); + } + fd_real_stderr = open(tmp_stderr2, O_RDONLY | O_BINARY); + if (fd_real_stderr == -1) { + cc_log("Failed opening %s: %s", tmp_stderr2, strerror(errno)); + failed(); + } + fd_result = open(tmp_stderr, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666); + if (fd_result == -1) { + cc_log("Failed opening %s: %s", tmp_stderr, strerror(errno)); + failed(); + } + copy_fd(fd_cpp_stderr, fd_result); + copy_fd(fd_real_stderr, fd_result); + close(fd_cpp_stderr); + close(fd_real_stderr); + close(fd_result); + tmp_unlink(tmp_stderr2); + free(tmp_stderr2); + } + + if (status != 0) { + int fd; + cc_log("Compiler gave exit status %d", status); + stats_update(STATS_STATUS); + + fd = open(tmp_stderr, O_RDONLY | O_BINARY); + if (fd != -1) { + /* We can output stderr immediately instead of rerunning the compiler. */ + copy_fd(fd, 2); + close(fd); + tmp_unlink(tmp_stderr); + + x_exit(status); + } + + tmp_unlink(tmp_stderr); + failed(); + } + + if (stat(output_obj, &st) != 0) { + cc_log("Compiler didn't produce an object file"); + stats_update(STATS_NOOUTPUT); + failed(); + } + if (st.st_size == 0) { + cc_log("Compiler produced an empty object file"); + stats_update(STATS_EMPTYOUTPUT); + failed(); + } + + if (x_stat(tmp_stderr, &st) != 0) { + stats_update(STATS_ERROR); + failed(); + } + /* cache stderr */ + if (!read_file(tmp_stderr, 0, &stderr_d, &stderr_l)) { + stats_update(STATS_ERROR); + failed(); + } + tmp_unlink(tmp_stderr); + + if (output_dia) { + if (x_stat(output_dia, &st) != 0) { + stats_update(STATS_ERROR); + failed(); + } + /* cache dia */ + if (!read_file(output_dia, 0, &dia_d, &dia_l)) { + stats_update(STATS_ERROR); + failed(); + } + } + + /* cache output */ + if (!read_file(output_obj, 0, &obj_d, &obj_l)) { + stats_update(STATS_ERROR); + failed(); + } + + if (generating_dependencies) { + if (!read_file(output_dep, 0, &dep_d, &dep_l)) { + stats_update(STATS_ERROR); + failed(); + } + } + + if (memccached_set(cached_key, obj_d, stderr_d, dia_d, dep_d, + obj_l, stderr_l, dia_l, dep_l) < 0) { + stats_update(STATS_ERROR); + failed(); + } + + cc_log("Storing %s in memcached", cached_key); + + stats_update(STATS_TOCACHE); + + /* Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can + * be done almost anywhere, but we might as well do it near the end as we + * save the stat call if we exit early. + */ + { + char *first_level_dir = dirname(stats_file); + if (create_cachedirtag(first_level_dir) != 0) { + cc_log("Failed to create %s/CACHEDIR.TAG (%s)\n", + first_level_dir, strerror(errno)); + stats_update(STATS_ERROR); + failed(); + } + free(first_level_dir); + + /* Remove any CACHEDIR.TAG on the cache_dir level where it was located in + * previous ccache versions. */ + if (getpid() % 1000 == 0) { + char *path = format("%s/CACHEDIR.TAG", conf->cache_dir); + x_unlink(path); + free(path); + } + } + + /* Everything OK. */ + send_cached_stderr(); + update_manifest_file(); + + free(tmp_stderr); + free(tmp_stdout); +} +#endif + // Find the object file name by running the compiler in preprocessor mode. // Returns the hash as a heap-allocated hex string. static struct file_hash * @@ -1408,6 +1719,7 @@ static void update_cached_result_globals(struct file_hash *hash) { char *object_name = format_hash_as_string(hash->hash, hash->size); + cached_key = strdup(object_name); cached_obj_hash = hash; cached_obj = get_path_in_cache(object_name, ".o"); cached_stderr = get_path_in_cache(object_name, ".stderr"); @@ -1599,6 +1911,11 @@ calculate_common_hash(struct args *args, struct mdfour *hash) static struct file_hash * calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) { +#if HAVE_LIBMEMCACHED + char *data; + size_t size; +#endif + if (direct_mode) { hash_delimiter(hash, "manifest version"); hash_int(hash, MANIFEST_VERSION); @@ -1791,7 +2108,27 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) } char *manifest_name = hash_result(hash); manifest_path = get_path_in_cache(manifest_name, ".manifest"); - free(manifest_name); + /* Check if the manifest file is there. */ + struct stat st; + if (stat(manifest_path, &st) != 0) { +#if HAVE_LIBMEMCACHED + void *cache = NULL; +#endif + cc_log("Manifest file %s not in cache", manifest_path); +#if HAVE_LIBMEMCACHED + if (strlen(conf->memcached_conf) > 0) { + cc_log("Getting %s from memcached", manifest_name); + cache = memccached_raw_get(manifest_name, &data, &size); + } + if (cache) { + cc_log("Added object file hash to %s", manifest_path); + write_file(data, manifest_path, size); + stats_update_size(size, 1); + free(cache); + } else +#endif + return NULL; + } cc_log("Looking for object file hash in %s", manifest_path); object_hash = manifest_get(conf, manifest_path); if (object_hash) { @@ -1828,8 +2165,13 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) // Try to return the compile result from cache. If we can return from cache // then this function exits with the correct status code, otherwise it returns. static void -from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) +from_fscache(enum fromcache_call_mode mode, bool put_object_in_manifest) { +#if HAVE_LIBMEMCACHED + char *data_obj, *data_stderr, *data_dia, *data_dep; + size_t size_obj, size_stderr, size_dia, size_dep; +#endif + // The user might be disabling cache hits. if (conf->recache) { return; @@ -1837,7 +2179,33 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) struct stat st; if (stat(cached_obj, &st) != 0) { +#if HAVE_LIBMEMCACHED + void *cache = NULL; +#endif cc_log("Object file %s not in cache", cached_obj); +#if HAVE_LIBMEMCACHED + if (strlen(conf->memcached_conf) > 0 && + !using_split_dwarf && + !generating_coverage) { + cc_log("Getting %s from memcached", cached_key); + cache = memccached_get(cached_key, + &data_obj, &data_stderr, &data_dia, &data_dep, + &size_obj, &size_stderr, &size_dia, &size_dep); + } + if (cache) { + put_data_in_cache(data_obj, size_obj, cached_obj); + if (size_stderr > 0) { + put_data_in_cache(data_stderr, size_stderr, cached_stderr); + } + if (size_dia > 0) { + put_data_in_cache(data_dia, size_dia, cached_dia); + } + if (size_dep > 0) { + put_data_in_cache(data_dep, size_dep, cached_dep); + } + memccached_free(cache); + } else +#endif return; } @@ -1947,6 +2315,97 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) x_exit(0); } +#ifdef HAVE_LIBMEMCACHED +/* + * Try to return the compile result from cache. If we can return from cache + * then this function exits with the correct status code, otherwise it returns. + */ +static void +from_memcached(enum fromcache_call_mode mode, bool put_object_in_manifest) +{ + bool produce_dep_file = false; + int ret; + void *cache; + char *data_obj, *data_stderr, *data_dia, *data_dep; + size_t size_obj, size_stderr, size_dia, size_dep; + + /* the user might be disabling cache hits */ + if (conf->recache || using_split_dwarf || generating_coverage) { + return; + } + + cc_log("Getting %s from memcached", cached_key); + cache = memccached_get(cached_key, + &data_obj, &data_stderr, &data_dia, &data_dep, + &size_obj, &size_stderr, &size_dia, &size_dep); + if (!cache) { + return; + } + + /* + * (If mode != FROMCACHE_DIRECT_MODE, the dependency file is created by + * gcc.) + */ + produce_dep_file = generating_dependencies && mode == FROMCACHE_DIRECT_MODE; + + if (!str_eq(output_obj, "/dev/null")) { + x_unlink(output_obj); + ret = write_file(data_obj, output_obj, size_obj); + } else { + ret = 0; + } + if (ret < 0) { + cc_log("Problem creating %s from %s", output_obj, cached_key); + failed(); + } + + if (produce_dep_file) { + x_unlink(output_dep); + ret = write_file(data_dep, output_dep, size_dep); + if (ret < 0) { + cc_log("Problem creating %s from %s", output_dep, cached_key); + failed(); + } + } + if (output_dia) { + x_unlink(output_dia); + ret = write_file(data_dia, output_dia, size_dia); + if (ret < 0) { + cc_log("Problem creating %s from %s", output_dia, cached_key); + failed(); + } + } + + if (generating_dependencies && mode == FROMCACHE_CPP_MODE) { + /* Store the dependency file in the cache. */ + cc_log("Does not support non direct mode"); + } + + /* Send the stderr, if any. */ + safe_write(2, data_stderr, size_stderr); + + if (put_object_in_manifest) { + update_manifest_file(); + } + + /* log the cache hit */ + switch (mode) { + case FROMCACHE_DIRECT_MODE: + cc_log("Succeeded getting cached result"); + stats_update(STATS_CACHEHIT_DIR); + break; + + case FROMCACHE_CPP_MODE: + cc_log("Succeeded getting cached result"); + stats_update(STATS_CACHEHIT_CPP); + break; + } + + /* and exit with the right status code */ + x_exit(0); +} +#endif + // Find the real compiler. We just search the PATH to find an executable of the // same name that isn't a link to ourselves. static void @@ -3059,6 +3518,19 @@ initialize(void) create_initial_config_file(conf, primary_config_path); } + from_cache = from_fscache; + to_cache = to_fscache; + +#ifdef HAVE_LIBMEMCACHED + if (strlen(conf->memcached_conf) > 0) { + memccached_init(conf->memcached_conf); + } + + if (conf->memcached_only) { + from_cache = from_memcached; + to_cache = to_memcached; + } +#endif exitfn_init(); exitfn_add_nullary(stats_flush); exitfn_add_nullary(clean_up_pending_tmp_files); @@ -3089,6 +3561,7 @@ cc_reset(void) free(output_dep); output_dep = NULL; free(output_cov); output_cov = NULL; free(output_dia); output_dia = NULL; + free(cached_key); cached_key = NULL; free(cached_obj_hash); cached_obj_hash = NULL; free(cached_obj); cached_obj = NULL; free(cached_dwo); cached_dwo = NULL; @@ -3096,6 +3569,7 @@ cc_reset(void) free(cached_dep); cached_dep = NULL; free(cached_cov); cached_cov = NULL; free(cached_dia); cached_dia = NULL; + free(manifest_name); manifest_name = NULL; free(manifest_path); manifest_path = NULL; time_of_compilation = 0; for (size_t i = 0; i < ignore_headers_len; i++) { @@ -3119,6 +3593,10 @@ cc_reset(void) free(stats_file); stats_file = NULL; output_is_precompiled_header = false; +#ifdef HAVE_LIBMEMCACHED + memccached_release(); +#endif + conf = conf_create(); using_split_dwarf = false; } @@ -3285,8 +3763,14 @@ ccache(int argc, char *argv[]) put_object_in_manifest = true; } - // If we can return from cache at this point then do. - from_cache(FROMCACHE_CPP_MODE, put_object_in_manifest); + /* don't hit memcached twice */ + if (conf->memcached_only && object_hash_from_manifest + && file_hashes_equal(object_hash_from_manifest, object_hash)) { + cc_log("Already searched for %s", cached_key); + } else { + // If we can return from cache at this point then do. + from_cache(FROMCACHE_CPP_MODE, put_object_in_manifest); + } if (conf->read_only) { cc_log("Read-only mode; running real compiler"); diff --git a/ccache.h b/ccache.h index 7b29bb8..1c1e38d 100644 --- a/ccache.h +++ b/ccache.h @@ -126,6 +126,8 @@ void cc_log_argv(const char *prefix, char **argv); void fatal(const char *format, ...) ATTR_FORMAT(printf, 1, 2) ATTR_NORETURN; void copy_fd(int fd_in, int fd_out); +int safe_write(int fd_out, const char *data, size_t length); +int write_file(const char *data, const char *dest, size_t length); int copy_file(const char *src, const char *dest, int compress_level); int move_file(const char *src, const char *dest, int compress_level); int move_uncompressed_file(const char *src, const char *dest, @@ -185,6 +187,23 @@ char *read_text_file(const char *path, size_t size_hint); char *subst_env_in_string(const char *str, char **errmsg); // ---------------------------------------------------------------------------- +// memccached.c + +int memccached_init(char *conf); +int memccached_raw_set(const char *key, const char* data, size_t len); +int memccached_set( + const char *key, + const char *out, const char *err, const char *dia, const char *dep, + size_t out_len, size_t err_len, size_t dia_len, size_t dep_len); +void *memccached_raw_get(const char *key, char **data, size_t *len); +void* memccached_get( + const char *key, + char **out, char **err, char **dia, char **dep, + size_t *out_len, size_t *err_len, size_t *dia_len, size_t *dep_len); +void memccached_free(void *blob); +int memccached_release(void); + +// ---------------------------------------------------------------------------- // stats.c void stats_update(enum stats stat); diff --git a/conf.c b/conf.c index cfa2874..bf4e365 100644 --- a/conf.c +++ b/conf.c @@ -329,11 +329,14 @@ conf_create(void) conf->log_file = x_strdup(""); conf->max_files = 0; conf->max_size = (uint64_t)5 * 1000 * 1000 * 1000; + conf->memcached_conf = x_strdup(""); + conf->memcached_only = false; conf->path = x_strdup(""); conf->prefix_command = x_strdup(""); conf->prefix_command_cpp = x_strdup(""); conf->read_only = false; conf->read_only_direct = false; + conf->read_only_memcached = false; conf->recache = false; conf->run_second_cpp = true; conf->sloppiness = 0; @@ -362,6 +365,7 @@ conf_free(struct conf *conf) free(conf->extra_files_to_hash); free(conf->ignore_headers_in_manifest); free(conf->log_file); + free(conf->memcached_conf); free(conf->path); free(conf->prefix_command); free(conf->prefix_command_cpp); @@ -594,6 +598,12 @@ conf_print_items(struct conf *conf, printer(s, conf->item_origins[find_conf("max_size")->number], context); free(s2); + reformat(&s, "memcached_conf = %s", conf->memcached_conf); + printer(s, conf->item_origins[find_conf("memcached_conf")->number], context); + + reformat(&s, "memcached_only = %s", bool_to_string(conf->memcached_only)); + printer(s, conf->item_origins[find_conf("memcached_only")->number], context); + reformat(&s, "path = %s", conf->path); printer(s, conf->item_origins[find_conf("path")->number], context); @@ -611,6 +621,11 @@ conf_print_items(struct conf *conf, printer(s, conf->item_origins[find_conf("read_only_direct")->number], context); + reformat(&s, "read_only_memcached = %s", + bool_to_string(conf->read_only_memcached)); + printer(s, conf->item_origins[find_conf("read_only_memcached")->number], + context); + reformat(&s, "recache = %s", bool_to_string(conf->recache)); printer(s, conf->item_origins[find_conf("recache")->number], context); diff --git a/conf.h b/conf.h index 232dcfd..1e22016 100644 --- a/conf.h +++ b/conf.h @@ -23,11 +23,14 @@ struct conf { char *log_file; unsigned max_files; uint64_t max_size; + char *memcached_conf; + bool memcached_only; char *path; char *prefix_command; char *prefix_command_cpp; bool read_only; bool read_only_direct; + bool read_only_memcached; bool recache; bool run_second_cpp; unsigned sloppiness; diff --git a/configure.ac b/configure.ac index a35fac0..7ef33e1 100644 --- a/configure.ac +++ b/configure.ac @@ -16,6 +16,7 @@ case $host in ;; esac +AC_SUBST(ccache_memcached) AC_SUBST(extra_libs) AC_SUBST(include_dev_mk) AC_SUBST(test_suites) @@ -84,6 +85,31 @@ HW_FUNC_ASPRINTF dnl Check if -lm is needed. AC_SEARCH_LIBS(cos, m) +AC_ARG_ENABLE(static, + [AS_HELP_STRING([--enable-static], + [enable static link])]) + +if test x${enable_static} != x; then + extra_ldflags="-static" +fi + +AC_ARG_ENABLE(memcached, + [AS_HELP_STRING([--enable-memcached], + [enable memcached as a cache backend])]) + +dnl enable-memcached: Check if -lmemcached is needed. +if test x${enable_memcached} != x; then + if test x${enable_static} != x; then + AC_CHECK_LIB(stdc++, __gxx_personality_v0,[]) + fi + AC_CHECK_LIB(pthread, pthread_once) + AC_CHECK_LIB(memcached, memcached,[],[ + echo ' WARNING: recent version libmemcached not found' + echo ' please install libmemcached > 1.0 with development files' + exit 1 + ]) + ccache_memcached='CCACHE_MEMCACHED=1 ' +fi dnl Check for zlib AC_ARG_WITH(bundled-zlib, diff --git a/confitems.gperf b/confitems.gperf index 531bc92..fd43765 100644 --- a/confitems.gperf +++ b/confitems.gperf @@ -26,15 +26,18 @@ limit_multiple, 15, ITEM(limit_multiple, float) log_file, 16, ITEM(log_file, env_string) max_files, 17, ITEM(max_files, unsigned) max_size, 18, ITEM(max_size, size) -path, 19, ITEM(path, env_string) -prefix_command, 20, ITEM(prefix_command, env_string) -prefix_command_cpp, 21, ITEM(prefix_command_cpp, env_string) -read_only, 22, ITEM(read_only, bool) -read_only_direct, 23, ITEM(read_only_direct, bool) -recache, 24, ITEM(recache, bool) -run_second_cpp, 25, ITEM(run_second_cpp, bool) -sloppiness, 26, ITEM(sloppiness, sloppiness) -stats, 27, ITEM(stats, bool) -temporary_dir, 28, ITEM(temporary_dir, env_string) -umask, 29, ITEM(umask, umask) -unify, 30, ITEM(unify, bool) +memcached_conf, 19, ITEM(memcached_conf, string) +memcached_only, 20, ITEM(memcached_only, bool) +path, 21, ITEM(path, env_string) +prefix_command, 22, ITEM(prefix_command, env_string) +prefix_command_cpp, 23, ITEM(prefix_command_cpp, env_string) +read_only, 24, ITEM(read_only, bool) +read_only_direct, 25, ITEM(read_only_direct, bool) +read_only_memcached, 26, ITEM(read_only_memcached, bool) +recache, 27, ITEM(recache, bool) +run_second_cpp, 28, ITEM(run_second_cpp, bool) +sloppiness, 29, ITEM(sloppiness, sloppiness) +stats, 30, ITEM(stats, bool) +temporary_dir, 31, ITEM(temporary_dir, env_string) +umask, 32, ITEM(umask, umask) +unify, 33, ITEM(unify, bool) diff --git a/confitems_lookup.c b/confitems_lookup.c index 7482557..b324dad 100644 --- a/confitems_lookup.c +++ b/confitems_lookup.c @@ -1,6 +1,6 @@ /* ANSI-C code produced by gperf version 3.0.4 */ /* Command-line: gperf confitems.gperf */ -/* Computed positions: -k'1-2' */ +/* Computed positions: -k'1,$' */ #if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \ @@ -31,7 +31,7 @@ #line 8 "confitems.gperf" struct conf_item; -/* maximum key range = 46, duplicates = 0 */ +/* maximum key range = 65, duplicates = 0 */ #ifdef __GNUC__ __inline @@ -45,34 +45,34 @@ confitems_hash (register const char *str, register unsigned int len) { static const unsigned char asso_values[] = { - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 13, 0, - 5, 10, 50, 0, 30, 20, 50, 0, 10, 20, - 5, 0, 0, 50, 5, 0, 10, 15, 50, 50, - 20, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50 + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 5, 20, + 5, 0, 30, 70, 30, 10, 70, 20, 25, 0, + 10, 70, 0, 70, 0, 0, 10, 0, 70, 70, + 70, 55, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70 }; - return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]]; + return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]]; } static @@ -87,91 +87,108 @@ confitems_get (register const char *str, register unsigned int len) { enum { - TOTAL_KEYWORDS = 31, + TOTAL_KEYWORDS = 34, MIN_WORD_LENGTH = 4, MAX_WORD_LENGTH = 26, - MIN_HASH_VALUE = 4, - MAX_HASH_VALUE = 49 + MIN_HASH_VALUE = 5, + MAX_HASH_VALUE = 69 }; static const struct conf_item wordlist[] = { {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -#line 29 "confitems.gperf" - {"path", 19, ITEM(path, env_string)}, - {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -#line 13 "confitems.gperf" - {"compiler", 3, ITEM(compiler, string)}, -#line 11 "confitems.gperf" - {"cache_dir", 1, ITEM(cache_dir, env_string)}, - {"",0,NULL,0,NULL}, -#line 15 "confitems.gperf" - {"compression", 5, ITEM(compression, bool)}, - {"",0,NULL,0,NULL}, -#line 17 "confitems.gperf" - {"cpp_extension", 7, ITEM(cpp_extension, string)}, -#line 14 "confitems.gperf" - {"compiler_check", 4, ITEM(compiler_check, string)}, -#line 37 "confitems.gperf" - {"stats", 27, ITEM(stats, bool)}, -#line 12 "confitems.gperf" - {"cache_dir_levels", 2, ITEM_V(cache_dir_levels, unsigned, dir_levels)}, -#line 16 "confitems.gperf" - {"compression_level", 6, ITEM(compression_level, unsigned)}, -#line 26 "confitems.gperf" - {"log_file", 16, ITEM(log_file, env_string)}, -#line 30 "confitems.gperf" - {"prefix_command", 20, ITEM(prefix_command, env_string)}, -#line 36 "confitems.gperf" - {"sloppiness", 26, ITEM(sloppiness, sloppiness)}, -#line 10 "confitems.gperf" - {"base_dir", 0, ITEM_V(base_dir, env_string, absolute_path)}, -#line 34 "confitems.gperf" - {"recache", 24, ITEM(recache, bool)}, -#line 31 "confitems.gperf" - {"prefix_command_cpp", 21, ITEM(prefix_command_cpp, env_string)}, -#line 32 "confitems.gperf" - {"read_only", 22, ITEM(read_only, bool)}, #line 40 "confitems.gperf" - {"unify", 30, ITEM(unify, bool)}, + {"stats", 30, ITEM(stats, bool)}, {"",0,NULL,0,NULL}, -#line 24 "confitems.gperf" - {"keep_comments_cpp", 14, ITEM(keep_comments_cpp, bool)}, +#line 37 "confitems.gperf" + {"recache", 27, ITEM(recache, bool)}, #line 28 "confitems.gperf" {"max_size", 18, ITEM(max_size, size)}, #line 27 "confitems.gperf" {"max_files", 17, ITEM(max_files, unsigned)}, +#line 39 "confitems.gperf" + {"sloppiness", 29, ITEM(sloppiness, sloppiness)}, {"",0,NULL,0,NULL}, -#line 33 "confitems.gperf" - {"read_only_direct", 23, ITEM(read_only_direct, bool)}, #line 19 "confitems.gperf" {"disable", 9, ITEM(disable, bool)}, +#line 10 "confitems.gperf" + {"base_dir", 0, ITEM_V(base_dir, env_string, absolute_path)}, #line 38 "confitems.gperf" - {"temporary_dir", 28, ITEM(temporary_dir, env_string)}, -#line 35 "confitems.gperf" - {"run_second_cpp", 25, ITEM(run_second_cpp, bool)}, + {"run_second_cpp", 28, ITEM(run_second_cpp, bool)}, {"",0,NULL,0,NULL}, #line 18 "confitems.gperf" {"direct_mode", 8, ITEM(direct_mode, bool)}, {"",0,NULL,0,NULL}, -#line 22 "confitems.gperf" - {"hash_dir", 12, ITEM(hash_dir, bool)}, -#line 21 "confitems.gperf" - {"hard_link", 11, ITEM(hard_link, bool)}, -#line 39 "confitems.gperf" - {"umask", 29, ITEM(umask, umask)}, +#line 33 "confitems.gperf" + {"prefix_command_cpp", 23, ITEM(prefix_command_cpp, env_string)}, +#line 32 "confitems.gperf" + {"prefix_command", 22, ITEM(prefix_command, env_string)}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, +#line 41 "confitems.gperf" + {"temporary_dir", 31, ITEM(temporary_dir, env_string)}, +#line 36 "confitems.gperf" + {"read_only_memcached", 26, ITEM(read_only_memcached, bool)}, +#line 42 "confitems.gperf" + {"umask", 32, ITEM(umask, umask)}, +#line 35 "confitems.gperf" + {"read_only_direct", 25, ITEM(read_only_direct, bool)}, + {"",0,NULL,0,NULL}, +#line 13 "confitems.gperf" + {"compiler", 3, ITEM(compiler, string)}, +#line 11 "confitems.gperf" + {"cache_dir", 1, ITEM(cache_dir, env_string)}, + {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, + {"",0,NULL,0,NULL}, +#line 26 "confitems.gperf" + {"log_file", 16, ITEM(log_file, env_string)}, +#line 31 "confitems.gperf" + {"path", 21, ITEM(path, env_string)}, + {"",0,NULL,0,NULL}, +#line 12 "confitems.gperf" + {"cache_dir_levels", 2, ITEM_V(cache_dir_levels, unsigned, dir_levels)}, +#line 24 "confitems.gperf" + {"keep_comments_cpp", 14, ITEM(keep_comments_cpp, bool)}, +#line 22 "confitems.gperf" + {"hash_dir", 12, ITEM(hash_dir, bool)}, #line 25 "confitems.gperf" {"limit_multiple", 15, ITEM(limit_multiple, float)}, {"",0,NULL,0,NULL}, +#line 15 "confitems.gperf" + {"compression", 5, ITEM(compression, bool)}, + {"",0,NULL,0,NULL}, +#line 17 "confitems.gperf" + {"cpp_extension", 7, ITEM(cpp_extension, string)}, +#line 29 "confitems.gperf" + {"memcached_conf", 19, ITEM(memcached_conf, string)}, + {"",0,NULL,0,NULL}, #line 23 "confitems.gperf" {"ignore_headers_in_manifest", 13, ITEM(ignore_headers_in_manifest, env_string)}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, #line 20 "confitems.gperf" - {"extra_files_to_hash", 10, ITEM(extra_files_to_hash, env_string)} + {"extra_files_to_hash", 10, ITEM(extra_files_to_hash, env_string)}, + {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, + {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, +#line 14 "confitems.gperf" + {"compiler_check", 4, ITEM(compiler_check, string)}, + {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, + {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, +#line 21 "confitems.gperf" + {"hard_link", 11, ITEM(hard_link, bool)}, +#line 43 "confitems.gperf" + {"unify", 33, ITEM(unify, bool)}, + {"",0,NULL,0,NULL}, +#line 16 "confitems.gperf" + {"compression_level", 6, ITEM(compression_level, unsigned)}, + {"",0,NULL,0,NULL}, +#line 34 "confitems.gperf" + {"read_only", 24, ITEM(read_only, bool)}, + {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, + {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, +#line 30 "confitems.gperf" + {"memcached_only", 20, ITEM(memcached_only, bool)} }; if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH) @@ -188,4 +205,4 @@ confitems_get (register const char *str, register unsigned int len) } return 0; } -static const size_t CONFITEMS_TOTAL_KEYWORDS = 31; +static const size_t CONFITEMS_TOTAL_KEYWORDS = 34; diff --git a/dump-memcached.py b/dump-memcached.py new file mode 100755 index 0000000..e7b2b0d --- /dev/null +++ b/dump-memcached.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import memcache +import struct +import sys +import os +import binascii + +""" +/* blob format for storing: + + char magic[4]; # 'CCH1', might change for other version of ccache + # ccache will erase the blob in memcached if wrong magic + uint32_t obj_len; # network endian + char *obj[obj_len]; + uint32_t stderr_len; # network endian + char *stderr[stderr_len]; + uint32_t dia_len; # network endian + char *dia[dia_len]; + uint32_t dep_len; # network endian + char *dep[dep_len]; + +*/ +""" +MEMCCACHE_MAGIC = 'CCH1' + +def get_blob(token): + return token[4:4+struct.unpack('!I', val[0:4])[0]] +MEMCCACHE_BIG = 'CCBM' + +""" +/* blob format for big values: + + char magic[4]; # 'CCBM' + uint32_t numkeys; # network endian + uint32_t hash_size; # network endian + uint32_t reserved; # network endian + uint32_t value_length; # network endian + + hash of include file ( bytes) + size of include file (4 bytes unsigned int) + ... + + + +*/ +""" +MEMCCACHE_BIG = 'CCBM' + +server = os.getenv("MEMCACHED_SERVERS", "localhost") +mc = memcache.Client(server.split(','), debug=1) + +key = sys.argv[1] +val = mc.get(key) +if val[0:4] == MEMCCACHE_BIG: + numkeys = struct.unpack('!I', val[4:8])[0] + assert struct.unpack('!I', val[8:12])[0] == 16 + assert struct.unpack('!I', val[12:16])[0] == 0 + size = struct.unpack('!I', val[16:20])[0] + val = val[20:] + buf = "" + while val: + md4 = val[0:16] + size = struct.unpack('!I', val[16:20])[0] + val = val[20:] + subkey = "%s-%d" % (binascii.hexlify(md4), size) + subval = mc.get(subkey) + if not subval: + print "%s not found" % subkey + buf = buf + subval + val = buf +if val: + magic = val[0:4] + if magic == MEMCCACHE_MAGIC: + val = val[4:] + obj = get_blob(val) + val = val[4+len(obj):] + stderr = get_blob(val) + val = val[4+len(stderr):] + dia = get_blob(val) + val = val[4+len(dia):] + dep = get_blob(val) + val = val[4+len(dep):] + assert len(val) == 0 + print "%s: %d %d %d %d" % (key, len(obj), len(stderr), len(dia), len(dep)) + else: + print "wrong magic" +else: + print "key missing" diff --git a/envtoconfitems.gperf b/envtoconfitems.gperf index 81d8444..00f64e0 100644 --- a/envtoconfitems.gperf +++ b/envtoconfitems.gperf @@ -27,12 +27,15 @@ LIMIT_MULTIPLE, "limit_multiple" LOGFILE, "log_file" MAXFILES, "max_files" MAXSIZE, "max_size" +MEMCACHED_CONF, "memcached_conf" +MEMCACHED_ONLY, "memcached_only" NLEVELS, "cache_dir_levels" PATH, "path" PREFIX, "prefix_command" PREFIX_CPP, "prefix_command_cpp" READONLY, "read_only" READONLY_DIRECT, "read_only_direct" +READONLY_MEMCACHED, "read_only_memcached" RECACHE, "recache" SLOPPINESS, "sloppiness" STATS, "stats" diff --git a/envtoconfitems_lookup.c b/envtoconfitems_lookup.c index 1265bd6..4608827 100644 --- a/envtoconfitems_lookup.c +++ b/envtoconfitems_lookup.c @@ -1,6 +1,6 @@ /* ANSI-C code produced by gperf version 3.0.4 */ /* Command-line: gperf envtoconfitems.gperf */ -/* Computed positions: -k'1,5' */ +/* Computed positions: -k'1,5,11' */ #if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \ @@ -31,7 +31,7 @@ #line 9 "envtoconfitems.gperf" struct env_to_conf_item; -/* maximum key range = 42, duplicates = 0 */ +/* maximum key range = 47, duplicates = 0 */ #ifdef __GNUC__ __inline @@ -45,38 +45,46 @@ envtoconfitems_hash (register const char *str, register unsigned int len) { static const unsigned char asso_values[] = { - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 20, 0, 0, 10, - 0, 44, 5, 15, 0, 44, 10, 25, 9, 0, - 5, 10, 5, 15, 10, 5, 44, 44, 44, 44, - 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 5, 0, 0, 5, + 40, 49, 20, 5, 0, 49, 20, 5, 0, 5, + 5, 0, 15, 0, 25, 0, 25, 49, 49, 49, + 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49 }; register int hval = len; switch (hval) { default: + hval += asso_values[(unsigned char)str[10]]; + /*FALLTHROUGH*/ + case 10: + case 9: + case 8: + case 7: + case 6: + case 5: hval += asso_values[(unsigned char)str[4]+1]; /*FALLTHROUGH*/ case 4: @@ -101,11 +109,11 @@ envtoconfitems_get (register const char *str, register unsigned int len) { enum { - TOTAL_KEYWORDS = 31, + TOTAL_KEYWORDS = 34, MIN_WORD_LENGTH = 2, - MAX_WORD_LENGTH = 15, + MAX_WORD_LENGTH = 18, MIN_HASH_VALUE = 2, - MAX_HASH_VALUE = 43 + MAX_HASH_VALUE = 48 }; static const struct env_to_conf_item wordlist[] = @@ -117,72 +125,76 @@ envtoconfitems_get (register const char *str, register unsigned int len) {"DIR", "cache_dir"}, #line 16 "envtoconfitems.gperf" {"CPP2", "run_second_cpp"}, - {"",""}, +#line 44 "envtoconfitems.gperf" + {"UNIFY", "unify"}, #line 19 "envtoconfitems.gperf" {"DIRECT", "direct_mode"}, #line 20 "envtoconfitems.gperf" {"DISABLE", "disable"}, -#line 17 "envtoconfitems.gperf" - {"COMMENTS", "keep_comments_cpp"}, -#line 31 "envtoconfitems.gperf" +#line 14 "envtoconfitems.gperf" + {"COMPRESS", "compression"}, +#line 33 "envtoconfitems.gperf" {"PATH", "path"}, -#line 41 "envtoconfitems.gperf" - {"UNIFY", "unify"}, -#line 32 "envtoconfitems.gperf" +#line 40 "envtoconfitems.gperf" + {"SLOPPINESS", "sloppiness"}, +#line 34 "envtoconfitems.gperf" {"PREFIX", "prefix_command"}, -#line 36 "envtoconfitems.gperf" - {"RECACHE", "recache"}, +#line 29 "envtoconfitems.gperf" + {"MAXSIZE", "max_size"}, +#line 28 "envtoconfitems.gperf" + {"MAXFILES", "max_files"}, + {"",""}, +#line 35 "envtoconfitems.gperf" + {"PREFIX_CPP", "prefix_command_cpp"}, + {"",""}, +#line 11 "envtoconfitems.gperf" + {"BASEDIR", "base_dir"}, #line 13 "envtoconfitems.gperf" {"COMPILERCHECK", "compiler_check"}, +#line 21 "envtoconfitems.gperf" + {"EXTENSION", "cpp_extension"}, +#line 22 "envtoconfitems.gperf" + {"EXTRAFILES", "extra_files_to_hash"}, {"",""}, -#line 33 "envtoconfitems.gperf" - {"PREFIX_CPP", "prefix_command_cpp"}, +#line 39 "envtoconfitems.gperf" + {"RECACHE", "recache"}, +#line 25 "envtoconfitems.gperf" + {"IGNOREHEADERS", "ignore_headers_in_manifest"}, #line 30 "envtoconfitems.gperf" - {"NLEVELS", "cache_dir_levels"}, + {"MEMCACHED_CONF", "memcached_conf"}, +#line 43 "envtoconfitems.gperf" + {"UMASK", "umask"}, + {"",""}, #line 27 "envtoconfitems.gperf" {"LOGFILE", "log_file"}, -#line 34 "envtoconfitems.gperf" +#line 36 "envtoconfitems.gperf" {"READONLY", "read_only"}, -#line 21 "envtoconfitems.gperf" - {"EXTENSION", "cpp_extension"}, -#line 40 "envtoconfitems.gperf" - {"UMASK", "umask"}, +#line 31 "envtoconfitems.gperf" + {"MEMCACHED_ONLY", "memcached_only"}, +#line 41 "envtoconfitems.gperf" + {"STATS", "stats"}, {"",""}, #line 24 "envtoconfitems.gperf" {"HASHDIR", "hash_dir"}, -#line 14 "envtoconfitems.gperf" - {"COMPRESS", "compression"}, - {"",""}, -#line 35 "envtoconfitems.gperf" - {"READONLY_DIRECT", "read_only_direct"}, - {"",""}, -#line 39 "envtoconfitems.gperf" +#line 23 "envtoconfitems.gperf" + {"HARDLINK", "hard_link"}, + {"",""}, {"",""}, {"",""}, +#line 42 "envtoconfitems.gperf" {"TEMPDIR", "temporary_dir"}, #line 15 "envtoconfitems.gperf" {"COMPRESSLEVEL", "compression_level"}, #line 26 "envtoconfitems.gperf" {"LIMIT_MULTIPLE", "limit_multiple"}, -#line 38 "envtoconfitems.gperf" - {"STATS", "stats"}, - {"",""}, -#line 29 "envtoconfitems.gperf" - {"MAXSIZE", "max_size"}, -#line 28 "envtoconfitems.gperf" - {"MAXFILES", "max_files"}, - {"",""}, #line 37 "envtoconfitems.gperf" - {"SLOPPINESS", "sloppiness"}, - {"",""}, -#line 11 "envtoconfitems.gperf" - {"BASEDIR", "base_dir"}, -#line 23 "envtoconfitems.gperf" - {"HARDLINK", "hard_link"}, - {"",""}, -#line 22 "envtoconfitems.gperf" - {"EXTRAFILES", "extra_files_to_hash"}, + {"READONLY_DIRECT", "read_only_direct"}, {"",""}, {"",""}, -#line 25 "envtoconfitems.gperf" - {"IGNOREHEADERS", "ignore_headers_in_manifest"} +#line 38 "envtoconfitems.gperf" + {"READONLY_MEMCACHED", "read_only_memcached"}, + {"",""}, {"",""}, {"",""}, +#line 32 "envtoconfitems.gperf" + {"NLEVELS", "cache_dir_levels"}, +#line 17 "envtoconfitems.gperf" + {"COMMENTS", "keep_comments_cpp"} }; if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH) @@ -199,4 +211,4 @@ envtoconfitems_get (register const char *str, register unsigned int len) } return 0; } -static const size_t ENVTOCONFITEMS_TOTAL_KEYWORDS = 31; +static const size_t ENVTOCONFITEMS_TOTAL_KEYWORDS = 34; diff --git a/memccached.c b/memccached.c new file mode 100644 index 0000000..38c44aa --- /dev/null +++ b/memccached.c @@ -0,0 +1,433 @@ +#include "ccache.h" + +#ifdef HAVE_LIBMEMCACHED + +#include +#include + +#define MEMCCACHE_MAGIC "CCH1" +#define MEMCCACHE_BIG "CCBM" + +#define MAX_VALUE_SIZE (1000 << 10) /* 1M with memcached overhead */ +#define SPLIT_VALUE_SIZE MAX_VALUE_SIZE + +/* status variables for memcached */ +static memcached_st *memc; + +int memccached_init(char *conf) +{ + memc = memcached(conf, strlen(conf)); + if (!memc) { + char errorbuf[1024]; + libmemcached_check_configuration(conf, strlen(conf), errorbuf, 1024); + cc_log("Problem creating memcached with conf %s:\n%s\n", conf, errorbuf); + return -1; + } + /* Consistent hashing delivers better distribution and allows servers to be + added to the cluster with minimal cache losses */ + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_DISTRIBUTION, + MEMCACHED_DISTRIBUTION_CONSISTENT); + return 0; +} + +/* blob format for big values: + + char magic[4]; # 'CCBM' + uint32_t numkeys; # network endian + uint32_t hash_size; # network endian + uint32_t reserved; # network endian + uint32_t value_length; # network endian + + hash of include file ( bytes) + size of include file (4 bytes unsigned int) + ... + + + + */ +static memcached_return_t memccached_big_set(memcached_st *ptr, + const char *key, + size_t key_length, + const char *value, + size_t value_length, + time_t expiration, + uint32_t flags) +{ + char *buf; + size_t buflen; + char *p; + int numkeys; + struct mdfour md; + char subkey[20]; + size_t n; + memcached_return_t ret; + size_t x; + + numkeys = (value_length + SPLIT_VALUE_SIZE - 1) / SPLIT_VALUE_SIZE; + buflen = 20 + 20 * numkeys; + buf = x_malloc(buflen); + p = buf; + + memcpy(p, MEMCCACHE_BIG, 4); + *((uint32_t *) (p + 4)) = htonl(numkeys); + *((uint32_t *) (p + 8)) = htonl(16); + *((uint32_t *) (p + 12)) = htonl(0); + *((uint32_t *) (p + 16)) = htonl(value_length); + p += 20; + + for (x = 0; x < value_length; x += n) { + size_t remain; + char *s; + + remain = value_length - x; + n = remain > SPLIT_VALUE_SIZE ? SPLIT_VALUE_SIZE : remain; + + mdfour_begin(&md); + mdfour_update(&md, (const unsigned char *) value + x, n); + mdfour_result(&md, (unsigned char *) subkey); + *((uint32_t *) (subkey + 16)) = htonl(n); + s = format_hash_as_string((const unsigned char *) subkey, n); + cc_log("memcached_mset %s %zu", s, n); + ret = memcached_set(ptr, s, strlen(s), value + x, n, + expiration, flags); + free(s); + if (ret) { + cc_log("Failed to set key in memcached: %s", + memcached_strerror(memc, ret)); + return ret; + } + + memcpy(p, subkey, 20); + p += 20; + } + + cc_log("memcached_set %.*s %zu (%zu)", (int) key_length, key, buflen, + value_length); + ret = memcached_set(ptr, key, key_length, buf, buflen, + expiration, flags); + free(buf); + return ret; +} + +static char *memccached_big_get(memcached_st *ptr, + const char *key, + size_t key_length, + const char *value, + size_t *value_length, + uint32_t *flags, + memcached_return_t *error) +{ + char *buf; + size_t buflen; + size_t totalsize; + char *p; + const char *v; + int numkeys; + char **keys; + bool *key_seen; + size_t *key_lengths; + size_t *value_offsets; + int *value_lengths; + memcached_return_t ret; + memcached_result_st *result; + int n; + int i; + + if (!value) { + value = memcached_get(ptr, key, key_length, value_length, flags, error); + if (!value) { + return NULL; + } + } + + p = (char *) value; + if (memcmp(p, MEMCCACHE_BIG, 4) != 0) { + return NULL; + } + numkeys = ntohl(*(uint32_t *) (p + 4)); + assert(ntohl(*(uint32_t *) (p + 8)) == 16); + assert(ntohl(*(uint32_t *) (p + 12)) == 0); + totalsize = ntohl(*(uint32_t *) (p + 16)); + p += 20; + + keys = x_malloc(sizeof(char *) * numkeys); + key_seen = x_malloc(sizeof(bool) * numkeys); + key_lengths = x_malloc(sizeof(size_t) * numkeys); + value_offsets = x_malloc(sizeof(size_t) * numkeys); + value_lengths = x_malloc(sizeof(int) * numkeys); + + buflen = 0; + for (i = 0; i < numkeys; i++) { + n = ntohl(*((uint32_t *) (p + 16))); + keys[i] = format_hash_as_string((const unsigned char *) p, n); + key_lengths[i] = strlen(keys[i]); + key_seen[i] = false; + cc_log("memcached_mget %.*s %d", (int) key_lengths[i], keys[i], n); + value_offsets[i] = buflen; + value_lengths[i] = n; + buflen += n; + p += 20; + } + assert(buflen == totalsize); + + buf = x_malloc(buflen); + + ret = memcached_mget(ptr, (const char *const *) keys, key_lengths, numkeys); + if (ret) { + cc_log("Failed to mget keys in memcached: %s", + memcached_strerror(memc, ret)); + for (i = 0; i < numkeys; i++) { + free(keys[i]); + } + free(keys); + free(key_lengths); + return NULL; + } + + result = NULL; + do { + const char *k; + size_t l; + + result = memcached_fetch_result(ptr, result, &ret); + if (ret == MEMCACHED_END) { + break; + } + if (ret) { + cc_log("Failed to get key in memcached: %s", + memcached_strerror(memc, ret)); + return NULL; + } + k = memcached_result_key_value(result); + l = memcached_result_key_length(result); + p = NULL; + for (i = 0; i < numkeys; i++) { + if (l != key_lengths[i]) { + continue; + } + if (memcmp(k, keys[i], l) == 0) { + p = buf + value_offsets[i]; + break; + } + } + if (!p) { + cc_log("Unknown key was returned: %s", k); + return NULL; + } + if (key_seen[i]) { + cc_log("Have already seen chunk: %s", k); + return NULL; + } + key_seen[i] = true; + n = memcached_result_length(result); + v = memcached_result_value(result); + if (n != value_lengths[i]) { + cc_log("Unexpected length was returned"); + return NULL; + } + memcpy(p, v, n); + } while (ret == MEMCACHED_SUCCESS); + + for (i = 0; i < numkeys; i++) { + if (!key_seen[i]) { + cc_log("Failed to get all %d chunks", numkeys); + return NULL; + } + } + cc_log("memcached_get %.*s %zu (%zu)", (int) key_length, key, *value_length, + buflen); + for (i = 0; i < numkeys; i++) { + free(keys[i]); + } + free(keys); + free(key_lengths); + free(value_offsets); + free(value_lengths); + + *value_length = buflen; + return buf; +} + +int memccached_raw_set(const char *key, const char *data, size_t len) +{ + memcached_return_t mret; + + mret = memcached_set(memc, key, strlen(key), data, len, 0, 0); + if (mret != MEMCACHED_SUCCESS) { + cc_log("Failed to move %s to memcached: %s", key, + memcached_strerror(memc, mret)); + return -1; + } + return 0; +} + +/* blob format for storing: + + char magic[4]; # 'CCH1', might change for other version of ccache + # ccache will erase the blob in memcached if wrong magic + uint32_t obj_len; # network endian + char *obj[obj_len]; + uint32_t stderr_len; # network endian + char *stderr[stderr_len]; + uint32_t dia_len; # network endian + char *dia[dia_len]; + uint32_t dep_len; # network endian + char *dep[dep_len]; + + */ +int memccached_set(const char *key, + const char *obj, + const char *stderr, + const char *dia, + const char *dep, + size_t obj_len, + size_t stderr_len, + size_t dia_len, + size_t dep_len) +{ + size_t buf_len = 4 + 4*4 + obj_len + stderr_len + dia_len + dep_len; + char *buf = x_malloc(buf_len); + char *ptr; + memcached_return_t mret; + + memcpy(buf, MEMCCACHE_MAGIC, 4); + ptr = buf + 4; + +#define PROCESS_ONE_BUFFER(src_ptr, src_len) \ + do { \ + *((uint32_t *)ptr) = htonl(src_len); \ + ptr += 4; \ + if (src_len > 0) { \ + memcpy(ptr, src_ptr, src_len); \ + } \ + ptr += src_len; \ + } while (false) + + PROCESS_ONE_BUFFER(obj, obj_len); + PROCESS_ONE_BUFFER(stderr, stderr_len); + PROCESS_ONE_BUFFER(dia, dia_len); + PROCESS_ONE_BUFFER(dep, dep_len); + +#undef PROCESS_ONE_BUFFER + + if (buf_len > MAX_VALUE_SIZE) { + mret = memccached_big_set(memc, key, strlen(key), buf, buf_len, 0, 0); + } else { + mret = memcached_set(memc, key, strlen(key), buf, buf_len, 0, 0); + } + + if (mret != MEMCACHED_SUCCESS) { + cc_log("Failed to move %s to memcached: %s", key, + memcached_strerror(memc, mret)); + return -1; + } + return 0; +} + +static void *memccached_prune(const char *key) +{ + cc_log("key from memcached has wrong data %s: pruning...", key); + /* don't really care whether delete failed */ + memcached_delete(memc, key, strlen(key), 0); + return NULL; +} + +void *memccached_raw_get(const char *key, char **data, size_t *size) +{ + memcached_return_t mret; + void *value; + size_t value_l; + + value = memcached_get(memc, key, strlen(key), &value_l, + NULL /*flags*/, &mret); + if (!value) { + cc_log("Failed to get key from memcached %s: %s", key, + memcached_strerror(memc, mret)); + return NULL; + } + *data = value; + *size = value_l; + return value; /* caller must free this when done with the ptr */ +} + +void *memccached_get(const char *key, + char **obj, + char **stderr, + char **dia, + char **dep, + size_t *obj_len, + size_t *stderr_len, + size_t *dia_len, + size_t *dep_len) +{ + memcached_return_t mret; + char *value, *ptr; + size_t value_l; + value = memcached_get(memc, key, strlen(key), &value_l, + NULL /*flags*/, &mret); + if (!value) { + cc_log("Failed to get key from memcached %s: %s", key, + memcached_strerror(memc, mret)); + return NULL; + } + if (value_l > 4 && memcmp(value, MEMCCACHE_BIG, 4) == 0) { + value = memccached_big_get(memc, key, strlen(key), value, &value_l, + NULL /*flags*/, &mret); + } + if (!value) { + cc_log("Failed to get key from memcached %s: %s", key, + memcached_strerror(memc, mret)); + return NULL; + } + if (value_l < 20 || memcmp(value, MEMCCACHE_MAGIC, 4) != 0) { + cc_log("wrong magic or length %.4s: %d", value, (int)value_l); + free(value); + return memccached_prune(key); + } + ptr = value; + /* skip the magic */ + ptr += 4; + value_l -= 4; + +#define PROCESS_ONE_BUFFER(dst_ptr, dst_len) \ + do { \ + if (value_l < 4) { \ + free(value); \ + cc_log("no more buffer for %s: %d", \ + #dst_ptr, (int)value_l); \ + return memccached_prune(key); \ + } \ + dst_len = ntohl(*((uint32_t *)ptr)); \ + ptr += 4; value_l -= 4; \ + if (value_l < dst_len) { \ + cc_log("no more buffer for %s: %d %d", \ + #dst_ptr, (int)value_l, (int) dst_len); \ + free(value); \ + return memccached_prune(key); \ + } \ + dst_ptr = ptr; \ + ptr += dst_len; value_l -= dst_len; \ + } while (false) + + PROCESS_ONE_BUFFER(*obj, *obj_len); + PROCESS_ONE_BUFFER(*stderr, *stderr_len); + PROCESS_ONE_BUFFER(*dia, *dia_len); + PROCESS_ONE_BUFFER(*dep, *dep_len); + +#undef PROCESS_ONE_BUFFER + + return value; /* caller must free this when done with the ptrs */ +} + +void memccached_free(void *blob) +{ + free(blob); +} + +int memccached_release(void) +{ + memcached_free(memc); + return 1; +} + +#endif /* HAVE_LIBMEMCACHED */ diff --git a/test.sh b/test.sh index 3e04157..ac3eb6d 100755 --- a/test.sh +++ b/test.sh @@ -43,6 +43,11 @@ test_failed() { $CCACHE -s echo echo "Test data and log file have been left in $TESTDIR" + tail -n 50 $CCACHE_LOGFILE + if [ ! -z $CCACHE_MEMCACHED_CONF ]; then + memstat --servers=localhost:22122 + kill %1 + fi exit 1 } -@@ -188,6 +193,8 @@ TEST() { - unset CCACHE_IGNOREHEADERS - unset CCACHE_LIMIT_MULTIPLE - unset CCACHE_LOGFILE -+ unset CCACHE_MEMCACHED_CONF -+ unset CCACHE_MEMCACHED_ONLY - unset CCACHE_NLEVELS - unset CCACHE_NOCPP2 - unset CCACHE_NOHASHDIR @@ -244,13 +251,13 @@ base_tests() { $CCACHE_COMPILE -c test1.c expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 - expect_stat 'files in cache' 1 + $CCACHE_NOFILES expect_stat 'files in cache' 1 expect_equal_object_files reference_test1.o test1.o $CCACHE_COMPILE -c test1.c expect_stat 'cache hit (preprocessed)' 1 expect_stat 'cache miss' 1 - expect_stat 'files in cache' 1 + $CCACHE_NOFILES expect_stat 'files in cache' 1 expect_equal_object_files reference_test1.o test1.o # ------------------------------------------------------------------------- @@ -259,7 +266,7 @@ base_tests() { $CCACHE_COMPILE -c test1.c -g expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 - expect_stat 'files in cache' 1 + $CCACHE_NOFILES expect_stat 'files in cache' 1 $CCACHE_COMPILE -c test1.c -g expect_stat 'cache hit (preprocessed)' 1 @@ -602,7 +609,7 @@ b" done expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 32 - expect_stat 'files in cache' 32 + $CCACHE_NOFILES expect_stat 'files in cache' 32 # ------------------------------------------------------------------------- TEST "Called for preprocessing" @@ -1366,6 +1373,52 @@ SUITE_masquerading() { # ============================================================================= +SUITE_memcached_SETUP() { + generate_code 1 test1.c +} + +SUITE_memcached() { + export CCACHE_MEMCACHED_CONF=--SERVER=localhost:22122 + memcached -p 22122 & + memcached_pid=$! + base_tests + kill $memcached_pid + unset CCACHE_MEMCACHED_CONF +} + +SUITE_memcached_only_SETUP() { + generate_code 1 test1.c +} + +SUITE_memcached_only() { + CCACHE_NOFILES=true + export CCACHE_MEMCACHED_CONF=--SERVER=localhost:22122 + export CCACHE_MEMCACHED_ONLY=1 + memcached -p 22122 & + memcached_pid=$! + base_tests + kill $memcached_pid + unset CCACHE_MEMCACHED_CONF + unset CCACHE_MEMCACHED_ONLY + unset CCACHE_NOFILES +} + +SUITE_memcached_socket_SETUP() { + generate_code 1 test1.c +} + +SUITE_memcached_socket() { + export CCACHE_MEMCACHED_CONF=--SOCKET=\"/tmp/memcached.$$\" + memcached -s /tmp/memcached.$$ & + memcached_pid=$! + base_tests + kill $memcached_pid + rm /tmp/memcached.$$ + unset CCACHE_MEMCACHED_CONF +} + +# ============================================================================= + SUITE_hardlink_PROBE() { touch file1 if ! ln file1 file2 >/dev/null 2>&1; then @@ -1539,7 +1592,7 @@ EOF test_failed "$dep_file missing" fi done - expect_stat 'files in cache' 12 + $CCACHE_NOFILES expect_stat 'files in cache' 12 # ------------------------------------------------------------------------- TEST "-Wp,-MD" @@ -3412,6 +3465,14 @@ upgrade input_charset " +if [ ! -z $CCACHE_MEMCACHED ]; then + all_suites="$all_suites +memcached +memcached_only +memcached_socket +" +fi + compiler_location=$(which $(echo "$COMPILER" | awk '{print $1}')) if [ "$compiler_location" = "$COMPILER" ]; then echo "Compiler: $COMPILER" diff --git a/test/test_conf.c b/test/test_conf.c index ea43e2e..d65372c 100644 --- a/test/test_conf.c +++ b/test/test_conf.c @@ -18,7 +18,7 @@ #include "framework.h" #include "util.h" -#define N_CONFIG_ITEMS 31 +#define N_CONFIG_ITEMS 34 static struct { char *descr; const char *origin; @@ -68,11 +68,14 @@ TEST(conf_create) CHECK_STR_EQ("", conf->log_file); CHECK_INT_EQ(0, conf->max_files); CHECK_INT_EQ((uint64_t)5 * 1000 * 1000 * 1000, conf->max_size); + CHECK_STR_EQ("", conf->memcached_conf); + CHECK(!conf->memcached_only); CHECK_STR_EQ("", conf->path); CHECK_STR_EQ("", conf->prefix_command); CHECK_STR_EQ("", conf->prefix_command_cpp); CHECK(!conf->read_only); CHECK(!conf->read_only_direct); + CHECK(!conf->read_only_memcached); CHECK(!conf->recache); CHECK(conf->run_second_cpp); CHECK_INT_EQ(0, conf->sloppiness); @@ -119,11 +122,14 @@ TEST(conf_read_valid_config) "log_file = $USER${USER} \n" "max_files = 17\n" "max_size = 123M\n" + "memcached_conf = --SERVER=localhost\n" + "memcached_only = true\n" "path = $USER.x\n" "prefix_command = x$USER\n" "prefix_command_cpp = y\n" "read_only = true\n" "read_only_direct = true\n" + "read_only_memcached = false\n" "recache = true\n" "run_second_cpp = false\n" "sloppiness = file_macro ,time_macros, include_file_mtime,include_file_ctime,file_stat_matches,pch_defines , no_system_headers \n" @@ -157,11 +163,14 @@ TEST(conf_read_valid_config) CHECK_STR_EQ_FREE1(format("%s%s", user, user), conf->log_file); CHECK_INT_EQ(17, conf->max_files); CHECK_INT_EQ(123 * 1000 * 1000, conf->max_size); + CHECK_STR_EQ("--SERVER=localhost", conf->memcached_conf); + CHECK(conf->memcached_only); CHECK_STR_EQ_FREE1(format("%s.x", user), conf->path); CHECK_STR_EQ_FREE1(format("x%s", user), conf->prefix_command); CHECK_STR_EQ("y", conf->prefix_command_cpp); CHECK(conf->read_only); CHECK(conf->read_only_direct); + CHECK(!conf->read_only_memcached); CHECK(conf->recache); CHECK(!conf->run_second_cpp); CHECK_INT_EQ(SLOPPY_INCLUDE_FILE_MTIME|SLOPPY_INCLUDE_FILE_CTIME| @@ -383,11 +392,14 @@ TEST(conf_print_items) "lf", 4711, 98.7 * 1000 * 1000, + "mc", + false, "p", "pc", "pcc", true, true, + false, true, .run_second_cpp = false, SLOPPY_FILE_MACRO|SLOPPY_INCLUDE_FILE_MTIME| @@ -433,11 +445,14 @@ TEST(conf_print_items) CHECK_STR_EQ("log_file = lf", received_conf_items[n++].descr); CHECK_STR_EQ("max_files = 4711", received_conf_items[n++].descr); CHECK_STR_EQ("max_size = 98.7M", received_conf_items[n++].descr); + CHECK_STR_EQ("memcached_conf = mc", received_conf_items[n++].descr); + CHECK_STR_EQ("memcached_only = false", received_conf_items[n++].descr); CHECK_STR_EQ("path = p", received_conf_items[n++].descr); CHECK_STR_EQ("prefix_command = pc", received_conf_items[n++].descr); CHECK_STR_EQ("prefix_command_cpp = pcc", received_conf_items[n++].descr); CHECK_STR_EQ("read_only = true", received_conf_items[n++].descr); CHECK_STR_EQ("read_only_direct = true", received_conf_items[n++].descr); + CHECK_STR_EQ("read_only_memcached = false", received_conf_items[n++].descr); CHECK_STR_EQ("recache = true", received_conf_items[n++].descr); CHECK_STR_EQ("run_second_cpp = false", received_conf_items[n++].descr); CHECK_STR_EQ("sloppiness = file_macro, include_file_mtime," diff --git a/upload-memcached.py b/upload-memcached.py new file mode 100755 index 0000000..bc489b0 --- /dev/null +++ b/upload-memcached.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +import memcache +import struct +import os +import hashlib + +""" +/* blob format for storing: + + char magic[4]; # 'CCH1', might change for other version of ccache + # ccache will erase the blob in memcached if wrong magic + uint32_t obj_len; # network endian + char *obj[obj_len]; + uint32_t stderr_len; # network endian + char *stderr[stderr_len]; + uint32_t dia_len; # network endian + char *dia[dia_len]; + uint32_t dep_len; # network endian + char *dep[dep_len]; + +*/ +""" +MEMCCACHE_MAGIC = 'CCH1' + +def set_blob(data): + return struct.pack('!I', len(data)) + str(data) +MEMCCACHE_BIG = 'CCBM' + +""" +/* blob format for big values: + + char magic[4]; # 'CCBM' + uint32_t numkeys; # network endian + uint32_t hash_size; # network endian + uint32_t reserved; # network endian + uint32_t value_length; # network endian + + hash of include file ( bytes) + size of include file (4 bytes unsigned int) + ... + + + +*/ +""" +MEMCCACHE_BIG = 'CCBM' + +MAX_VALUE_SIZE = 1000 << 10 # 1M with memcached overhead +SPLIT_VALUE_SIZE = MAX_VALUE_SIZE + +server = os.getenv("MEMCACHED_SERVERS", "localhost") +mc = memcache.Client(server.split(','), debug=1) + +ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.ccache")) +filelist = [] +for dirpath, dirnames, filenames in os.walk(ccache): + # sort by modification time, most recently used last + for filename in filenames: + stat = os.stat(os.path.join(dirpath, filename)) + filelist.append((stat.st_mtime, dirpath, filename)) +filelist.sort() +files = blobs = chunks = objects = manifest = 0 +for mtime, dirpath, filename in filelist: + dirname = dirpath.replace(ccache + os.path.sep, "") + if filename == "CACHEDIR.TAG": + # ignore these + files = files + 1 + else: + (base, ext) = os.path.splitext(filename) + if ext == '.o': + objects = objects + 1 + key = "".join(list(os.path.split(dirname)) + [base]) + def read_file(path): + return os.path.exists(path) and open(path).read() or "" + obj = read_file(os.path.join(dirpath, filename)) + stderr = read_file(os.path.join(dirpath, base) + '.stderr') + dia = read_file(os.path.join(dirpath, base) + '.dia') + dep = read_file(os.path.join(dirpath, base) + '.d') + + print "%s: %d %d %d %d" % (key, len(obj), len(stderr), len(dia), len(dep)) + val = MEMCCACHE_MAGIC + val += set_blob(obj) + val += set_blob(stderr) + val += set_blob(dia) + val += set_blob(dep) + if len(val) > MAX_VALUE_SIZE: + numkeys = (len(val) + SPLIT_VALUE_SIZE - 1) / SPLIT_VALUE_SIZE + buf = MEMCCACHE_BIG + buf += struct.pack('!I', numkeys) + buf += struct.pack('!I', 16) + buf += struct.pack('!I', 0) + buf += struct.pack('!I', len(val)) + def splitchunks(s, n): + """Produce `n`-character chunks from `s`.""" + for start in range(0, len(s), n): + yield s[start:start+n] + valmap = {} + for subval in splitchunks(val, SPLIT_VALUE_SIZE): + subhash = hashlib.new('md4') + subhash.update(subval) + buf += subhash.digest() + struct.pack('!I', len(subval)) + subkey = "%s-%d" % (subhash.hexdigest(), len(subval)) + print "# %s: chunk %d" % (subkey, len(subval)) + #mc.set(subkey, subval) + valmap[subkey] = subval + chunks = chunks + 1 + mc.set_multi(valmap) + mc.set(key, buf) + else: + mc.set(key, val) + files = files + 1 + blobs = blobs + 1 + elif ext == '.stderr' or ext == '.d' or ext == '.dia': + # was added above + files = files + 1 + elif ext == '.manifest': + manifest = manifest + 1 + key = "".join(list(os.path.split(dirname)) + [base]) + val = open(os.path.join(dirpath, filename)).read() or None + if val: + print "%s: manifest %d" % (key, len(val)) + mc.set(key, val, 0, 0) + files = files + 1 + blobs = blobs + 1 +print "%d files, %d objects (%d manifest) = %d blobs (%d chunks)" % (files, objects, manifest, blobs, chunks) diff --git a/util.c b/util.c index f048d97..6059f25 100644 --- a/util.c +++ b/util.c @@ -388,6 +388,75 @@ copy_file(const char *src, const char *dest, int compress_level) return -1; } +// Write data to a fd. +int safe_write(int fd_out, const char *data, size_t length) +{ + size_t written = 0; + do { + int ret; + ret = write(fd_out, data + written, length - written); + if (ret < 0) { + if (errno != EAGAIN && errno != EINTR) { + return ret; + } + } else { + written += ret; + } + } while (written < length); + return 0; +} + +// Write data to a file. +int write_file(const char *data, const char *dest, size_t length) +{ + int fd_out; + char *tmp_name; + int ret; + int saved_errno = 0; + + tmp_name = x_strdup(dest); + fd_out = create_tmp_fd(&tmp_name); + if (fd_out < 0) { + tmp_unlink(tmp_name); + free(tmp_name); + return -1; + } + + ret = safe_write(fd_out, data, length); + if (ret < 0) { + saved_errno = errno; + cc_log("write error: %s", strerror(saved_errno)); + goto error; + } + +#ifndef _WIN32 + fchmod(fd_out, 0666 & ~get_umask()); +#endif + + /* the close can fail on NFS if out of space */ + if (close(fd_out) == -1) { + saved_errno = errno; + cc_log("close error: %s", strerror(saved_errno)); + goto error; + } + + if (x_rename(tmp_name, dest) == -1) { + saved_errno = errno; + cc_log("rename error: %s", strerror(saved_errno)); + goto error; + } + + free(tmp_name); + return 0; + +error: + close(fd_out); + tmp_unlink(tmp_name); + free(tmp_name); + errno = saved_errno; + return -1; +} + // Run copy_file() and, if successful, delete the source file. int move_file(const char *src, const char *dest, int compress_level)