Index: head/devel/hadoop2/Makefile =================================================================== --- head/devel/hadoop2/Makefile (revision 523701) +++ head/devel/hadoop2/Makefile (revision 523702) @@ -1,129 +1,121 @@ # Created by: Dmitry Sivachenko # $FreeBSD$ # Please do not submit untested updates. Be sure to start hadoop in # distributed mode and to run few map/reduce. Be sure there are no exception # in any of it's log files. This version was tested under the load and # no problems encountered so far. Thanks. PORTNAME= hadoop PORTVERSION= 2.7.2 -PORTREVISION= 2 +PORTREVISION= 3 CATEGORIES= devel java MASTER_SITES= APACHE/${PORTNAME}/common/hadoop-${PORTVERSION} \ LOCAL/demon/:maven \ http://archive.apache.org/dist/tomcat/tomcat-6/v${TOMCAT_VERSION}/bin/:tomcat \ LOCAL/demon/:jetty PKGNAMESUFFIX= 2 DISTNAME= ${PORTNAME}-${PORTVERSION}-src DISTFILES= ${DISTNAME}${EXTRACT_SUFX} FreeBSD-hadoop2-${PORTVERSION}-maven-repository.tar.gz:maven apache-tomcat-${TOMCAT_VERSION}.tar.gz:tomcat jetty-6.1.14.zip:jetty DIST_SUBDIR= hadoop EXTRACT_ONLY= ${DISTNAME}${EXTRACT_SUFX} FreeBSD-hadoop2-${PORTVERSION}-maven-repository.tar.gz jetty-6.1.14.zip MAINTAINER= kbowling@FreeBSD.org COMMENT= Apache Map/Reduce framework LICENSE= APACHE20 -BROKEN_SSL= openssl -BROKEN_SSL_REASON_openssl= incomplete definition of type 'struct evp_cipher_ctx_st' - BUILD_DEPENDS= ${LOCALBASE}/share/java/maven3/bin/mvn:devel/maven3 \ cmake:devel/cmake \ ${LOCALBASE}/protobuf25/bin/protoc:devel/protobuf25 LIB_DEPENDS= libsnappy.so:archivers/snappy RUN_DEPENDS= bash:shells/bash \ ssid:sysutils/ssid CONFLICTS_INSTALL= hadoop-1* yarn USES= cpe shebangfix CPE_VENDOR= apache USE_JAVA= yes USE_LDCONFIG= yes SHEBANG_FILES= hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh MAKE_ENV+= JAVA_HOME=${JAVA_HOME} HADOOP_PROTOC_PATH=${LOCALBASE}/protobuf25/bin/protoc OPTIONS_DEFINE= EXAMPLES TOMCAT_VERSION= 6.0.44 HADOOP_DIST= ${WRKSRC}/hadoop-dist/target/hadoop-${PORTVERSION} HADOOP_LOGDIR= /var/log/hadoop HADOOP_RUNDIR= /var/run/hadoop HDFS_USER= hdfs MAPRED_USER= mapred HADOOP_GROUP= hadoop USERS= ${HDFS_USER} ${MAPRED_USER} GROUPS= ${HADOOP_GROUP} SUB_FILES= hadoop-layout.sh httpfs-env.sh kms-env.sh USE_RC_SUBR= historyserver nodemanager resourcemanager webappproxyserver datanode namenode secondarynamenode journalnode zkfc PLIST_SUB= PORTVERSION="${PORTVERSION}" \ HADOOP_LOGDIR="${HADOOP_LOGDIR}" \ HADOOP_RUNDIR="${HADOOP_RUNDIR}" \ HDFS_USER="${HDFS_USER}" \ MAPRED_USER="${MAPRED_USER}" \ HADOOP_GROUP="${HADOOP_GROUP}" SUB_LIST= HDFS_USER="${HDFS_USER}" \ MAPRED_USER="${MAPRED_USER}" \ HADOOP_GROUP="${HADOOP_GROUP}" \ JAVA_HOME="${JAVA_HOME}" \ HADOOP_LOGDIR="${HADOOP_LOGDIR}" \ HADOOP_RUNDIR="${HADOOP_RUNDIR}" .include - -.if ${SSL_DEFAULT} == base -BROKEN_FreeBSD_12= incomplete definition of type 'struct evp_cipher_ctx_st' -BROKEN_FreeBSD_13= incomplete definition of type 'struct evp_cipher_ctx_st' -.endif post-patch: ${REINPLACE_CMD} -e "s#/bin/bash#${LOCALBASE}/bin/bash#" ${WRKSRC}/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java ${WRKSRC}/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ${WRKSRC}/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ${WRKSRC}/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java ${WRKSRC}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh do-build: ${MKDIR} ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads ${CP} ${DISTDIR}/${DIST_SUBDIR}/apache-tomcat-${TOMCAT_VERSION}.tar.gz ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads/ ${MKDIR} ${WRKSRC}/hadoop-common-project/hadoop-kms/downloads ${CP} ${DISTDIR}/${DIST_SUBDIR}/apache-tomcat-${TOMCAT_VERSION}.tar.gz ${WRKSRC}/hadoop-common-project/hadoop-kms/downloads/ cd ${WRKSRC} && ${SETENV} ${MAKE_ENV} ${LOCALBASE}/share/java/maven3/bin/mvn -Dmaven.repo.local=${WRKDIR}/m2 --offline package -Pdist,native -DskipTests -Drequire.snappy -Dsnappy.prefix=${LOCALBASE} -Drequire.openssl post-build: ${RM} ${HADOOP_DIST}/etc/hadoop/*.cmd ${MV} ${HADOOP_DIST}/bin/rcc ${HADOOP_DIST}/bin/hadoop-rcc # With jetty-6.1.26 tasktracker's threads hung with the following error: # org.mortbay.io.nio.SelectorManager$SelectSet@abdcc1c JVM BUG(s) - injecting delay 59 times # See https://issues.apache.org/jira/browse/MAPREDUCE-2386 .for dir in share/hadoop/common/lib share/hadoop/hdfs/lib share/hadoop/yarn/lib share/hadoop/tools/lib ${RM} ${HADOOP_DIST}/${dir}/jetty-util-6.1.26.jar ${HADOOP_DIST}/${dir}/jetty-6.1.26.jar ${CP} ${WRKDIR}/jetty-6.1.14/lib/jetty-6.1.14.jar ${WRKDIR}/jetty-6.1.14/lib/jetty-util-6.1.14.jar ${HADOOP_DIST}/${dir}/ .endfor .for dir in share/hadoop/httpfs/tomcat/webapps/webhdfs/WEB-INF/lib share/hadoop/kms/tomcat/webapps/kms/WEB-INF/lib ${RM} ${HADOOP_DIST}/${dir}/jetty-util-6.1.26.jar ${CP} ${WRKDIR}/jetty-6.1.14/lib/jetty-util-6.1.14.jar ${HADOOP_DIST}/${dir}/ .endfor do-install: cd ${HADOOP_DIST}/bin && ${INSTALL_SCRIPT} hadoop hdfs mapred hadoop-rcc yarn ${STAGEDIR}${PREFIX}/bin/ cd ${HADOOP_DIST} && ${COPYTREE_BIN} "libexec sbin" ${STAGEDIR}${PREFIX}/ "! -name *.cmd" cd ${HADOOP_DIST}/include && ${INSTALL_DATA} *h ${STAGEDIR}${PREFIX}/include/ cd ${HADOOP_DIST}/lib/native && ${INSTALL_DATA} *.a ${STAGEDIR}${PREFIX}/lib/ cd ${HADOOP_DIST}/lib/native && ${INSTALL_DATA} libhadoop.so.1.0.0 ${STAGEDIR}${PREFIX}/lib/libhadoop.so.1 cd ${HADOOP_DIST}/lib/native && ${INSTALL_DATA} libhdfs.so.0.0.0 ${STAGEDIR}${PREFIX}/lib/libhdfs.so.0 ${LN} -sf libhdfs.so.0 ${STAGEDIR}${PREFIX}/lib/libhdfs.so ${LN} -sf libhadoop.so.1 ${STAGEDIR}${PREFIX}/lib/libhadoop.so cd ${HADOOP_DIST}/share/hadoop && ${COPYTREE_SHARE} "*" ${STAGEDIR}${DATADIR}/ "! -name *-sources.jar -and ! -name sources" ${CHMOD} a+x ${STAGEDIR}${DATADIR}/kms/tomcat/bin/*.sh ${STAGEDIR}${DATADIR}/httpfs/tomcat/bin/*.sh ${MKDIR} ${STAGEDIR}${EXAMPLESDIR}/conf cd ${HADOOP_DIST}/etc/hadoop && ${COPYTREE_SHARE} "*" ${STAGEDIR}${EXAMPLESDIR}/conf/ ${INSTALL_DATA} ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs/target/classes/hdfs-default.xml ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/classes/httpfs-default.xml ${WRKSRC}/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes/yarn-default.xml ${WRKSRC}/hadoop-common-project/hadoop-common/target/classes/core-default.xml ${WRKSRC}/hadoop-tools/hadoop-distcp/target/classes/distcp-default.xml ${WRKSRC}/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/classes/mapred-default.xml ${STAGEDIR}/${EXAMPLESDIR}/ ${INSTALL_DATA} ${WRKDIR}/hadoop-layout.sh ${STAGEDIR}${PREFIX}/libexec/ ${MKDIR} ${STAGEDIR}${ETCDIR} ${INSTALL_DATA} ${WRKDIR}/httpfs-env.sh ${STAGEDIR}${PREFIX}/etc/hadoop/ ${INSTALL_DATA} ${WRKDIR}/kms-env.sh ${STAGEDIR}${PREFIX}/etc/hadoop/ ${MKDIR} ${STAGEDIR}${HADOOP_LOGDIR} ${MKDIR} ${STAGEDIR}${HADOOP_RUNDIR} .include Index: head/devel/hadoop2/files/patch-openssl111 =================================================================== --- head/devel/hadoop2/files/patch-openssl111 (nonexistent) +++ head/devel/hadoop2/files/patch-openssl111 (revision 523702) @@ -0,0 +1,134 @@ +--- hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -30,6 +30,11 @@ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); + static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *); + static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *); + static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int); ++static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int); ++static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++static int (*dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *); ++#endif + static int (*dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, const EVP_CIPHER *, \ + ENGINE *, const unsigned char *, const unsigned char *, int); + static int (*dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, unsigned char *, \ +@@ -46,6 +51,11 @@ typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); + typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *); + typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *); + typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int); ++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int); ++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *); ++#endif + typedef int (__cdecl *__dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, \ + const EVP_CIPHER *, ENGINE *, const unsigned char *, \ + const unsigned char *, int); +@@ -60,6 +70,11 @@ static __dlsym_EVP_CIPHER_CTX_free dlsym_EVP_CIPHER_CTX_free; + static __dlsym_EVP_CIPHER_CTX_cleanup dlsym_EVP_CIPHER_CTX_cleanup; + static __dlsym_EVP_CIPHER_CTX_init dlsym_EVP_CIPHER_CTX_init; + static __dlsym_EVP_CIPHER_CTX_set_padding dlsym_EVP_CIPHER_CTX_set_padding; ++static __dlsym_EVP_CIPHER_CTX_test_flags dlsym_EVP_CIPHER_CTX_test_flags; ++static __dlsym_EVP_CIPHER_CTX_block_size dlsym_EVP_CIPHER_CTX_block_size; ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++static __dlsym_EVP_CIPHER_CTX_encrypting dlsym_EVP_CIPHER_CTX_encrypting; ++#endif + static __dlsym_EVP_CipherInit_ex dlsym_EVP_CipherInit_ex; + static __dlsym_EVP_CipherUpdate dlsym_EVP_CipherUpdate; + static __dlsym_EVP_CipherFinal_ex dlsym_EVP_CipherFinal_ex; +@@ -114,6 +129,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + "EVP_CIPHER_CTX_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \ + "EVP_CIPHER_CTX_set_padding"); ++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \ ++ "EVP_CIPHER_CTX_test_flags"); ++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl, \ ++ "EVP_CIPHER_CTX_block_size"); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl, \ ++ "EVP_CIPHER_CTX_encrypting"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \ + "EVP_CipherInit_ex"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherUpdate, env, openssl, \ +@@ -135,6 +158,17 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_set_padding, \ + dlsym_EVP_CIPHER_CTX_set_padding, env, \ + openssl, "EVP_CIPHER_CTX_set_padding"); ++ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_test_flags, \ ++ dlsym_EVP_CIPHER_CTX_test_flags, env, \ ++ openssl, "EVP_CIPHER_CTX_test_flags"); ++ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size, \ ++ dlsym_EVP_CIPHER_CTX_block_size, env, \ ++ openssl, "EVP_CIPHER_CTX_block_size"); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting, \ ++ dlsym_EVP_CIPHER_CTX_encrypting, env, \ ++ openssl, "EVP_CIPHER_CTX_encrypting"); ++#endif + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \ + env, openssl, "EVP_CipherInit_ex"); + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherUpdate, dlsym_EVP_CipherUpdate, \ +@@ -253,14 +287,18 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init + static int check_update_max_output_len(EVP_CIPHER_CTX *context, int input_len, + int max_output_len) + { +- if (context->flags & EVP_CIPH_NO_PADDING) { ++ if ( dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) { + if (max_output_len >= input_len) { + return 1; + } + return 0; + } else { +- int b = context->cipher->block_size; ++ int b = dlsym_EVP_CIPHER_CTX_block_size(context); ++#if OPENSSL_VERSION_NUMBER < 0x10100000L + if (context->encrypt) { ++#else ++ if (dlsym_EVP_CIPHER_CTX_encrypting(context)) { ++#endif + if (max_output_len >= input_len + b - 1) { + return 1; + } +@@ -307,10 +345,10 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update + static int check_doFinal_max_output_len(EVP_CIPHER_CTX *context, + int max_output_len) + { +- if (context->flags & EVP_CIPH_NO_PADDING) { ++ if ( dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) { + return 1; + } else { +- int b = context->cipher->block_size; ++ int b = dlsym_EVP_CIPHER_CTX_block_size(context); + if (max_output_len >= b) { + return 1; + } +--- hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc ++++ hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc +@@ -420,6 +420,7 @@ namespace HadoopPipes { + } + + string createDigest(string &password, string& msg) { ++#if OPENSSL_VERSION_NUMBER < 0x10100000L + HMAC_CTX ctx; + unsigned char digest[EVP_MAX_MD_SIZE]; + HMAC_Init(&ctx, (const unsigned char *)password.c_str(), +@@ -428,7 +429,16 @@ namespace HadoopPipes { + unsigned int digestLen; + HMAC_Final(&ctx, digest, &digestLen); + HMAC_cleanup(&ctx); +- ++#else ++ HMAC_CTX *ctx = HMAC_CTX_new(); ++ unsigned char digest[EVP_MAX_MD_SIZE]; ++ HMAC_Init_ex(ctx, (const unsigned char *)password.c_str(), ++ password.length(), EVP_sha1(), NULL); ++ HMAC_Update(ctx, (const unsigned char *)msg.c_str(), msg.length()); ++ unsigned int digestLen; ++ HMAC_Final(ctx, digest, &digestLen); ++ HMAC_CTX_free(ctx); ++#endif + //now apply base64 encoding + BIO *bmem, *b64; + BUF_MEM *bptr; + Property changes on: head/devel/hadoop2/files/patch-openssl111 ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +yes \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property