From b1907161a56f05dbd32100a387420b7dc64369c0 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Jun 2021 20:10:57 +0800 Subject: [PATCH 1/2] match centos8 --- .../apache/Dockerfile-hadoop-3.1.1 | 23 ++++++++++++++++-- .../apache/Dockerfile-hbase-2.1.0 | 24 +++++++++++++++---- .../apache/Dockerfile-hive-3.0.0 | 24 ++++++++++++++++--- .../apache/Dockerfile-spark-2.3.0 | 7 +++++- .../apache/Dockerfile-zookeeper-3.4.6 | 6 ++++- 5 files changed, 73 insertions(+), 11 deletions(-) diff --git a/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 b/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 index ba08348..222424c 100644 --- a/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 +++ b/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 @@ -1,5 +1,10 @@ FROM arm64v8/centos:1911 - +WORKDIR /etc/yum.repos.d/ +RUN sed -i "s/enabled=0/enabled=1/g" *repo \ + && mv CentOS-Media.repo CentOS-Media.repo.bak \ + && yum clean all \ + && yum makecache \ + && yum install -y patch unzip diffutils libtirpc-devel WORKDIR /root/ #RUN cp -r /etc/yum.repos.d /etc/yum.repos.d-bak \ # && rm /etc/yum.repos.d/* -rf \ @@ -41,10 +46,24 @@ RUN cd /opt/ \ && ./bootstrap \ && make -j8 \ && make install \ - && yum install -y openssl-devel zlib-devel automake libtool protobuf protobuf-devel \ + && yum install -y openssl-devel zlib-devel automake libtool \ && cd /opt/ \ # && wget https://github.com/google/snappy/archive/1.1.7.tar.gz \ && wget http://archive-server:50050/cache-data/BigData/Apache/hadoop/1.1.7.tar.gz \ + && yum install -y patch unzip \ +# && wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protobuf-2.5.0.tar.gz \ + && tar -zxf protobuf-2.5.0.tar.gz \ +# && wget https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/bigdata/Patch/protoc.zip \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protoc.zip \ + && unzip protoc.zip \ + && cp ./protoc/protoc.patch ./protobuf-2.5.0/src/google/protobuf/stubs/ \ + && cd protobuf-2.5.0/src/google/protobuf/stubs/ \ + && patch -p1 < protoc.patch \ + && cd ../../../../ \ + && ./autogen.sh && ./configure CFLAGS='-fsigned-char' && make -j8 && make install \ + && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc \ + && cd ../ \ && tar -zxf 1.1.7.tar.gz \ && cd snappy-1.1.7/ \ && sed -i "s/OFF/ON/g" /opt/snappy-1.1.7/CMakeLists.txt \ diff --git a/BigData/CentOS8.1.1911/apache/Dockerfile-hbase-2.1.0 b/BigData/CentOS8.1.1911/apache/Dockerfile-hbase-2.1.0 index 8fff839..c4c86c5 100644 --- a/BigData/CentOS8.1.1911/apache/Dockerfile-hbase-2.1.0 +++ b/BigData/CentOS8.1.1911/apache/Dockerfile-hbase-2.1.0 @@ -1,5 +1,9 @@ FROM arm64v8/centos:1911 - +WORKDIR /etc/yum.repos.d/ +RUN sed -i "s/enabled=0/enabled=1/g" *repo \ + && mv CentOS-Media.repo CentOS-Media.repo.bak \ + && yum clean all \ + && yum makecache WORKDIR /root/ #RUN cp -r /etc/yum.repos.d /etc/yum.repos.d-bak \ # && rm /etc/yum.repos.d/* -rf \ @@ -14,7 +18,7 @@ RUN yum install -y boost.aarch64 boost-devel.aarch64 make cmake wget openssl-dev && mv /usr/bin/g++ /usr/bin/g++-impl \ && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ && chmod +x /usr/bin/gcc /usr/bin/g++ \ - && yum install vim wget protobuf protobuf-devel -y \ + && yum install vim wget -y \ && mkdir -p /opt/src \ && wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_8u252b09.tar.gz \ #https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_aarch64_linux_hotspot_8u252b09.tar.gz @@ -32,8 +36,20 @@ ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.5.4 ENV PATH=$MAVEN_HOME/bin:$PATH #RUN wget http://archive.apache.org/dist/hbase/2.1.0/hbase-2.1.0-src.tar.gz \ RUN wget http://archive-server:50050/cache-data/BigData/Apache/hbase/hbase-2.1.0-src.tar.gz \ - && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/bin/protoc \ -# && cd /opt \ + && yum install -y patch unzip diffutils \ +# && wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protobuf-2.5.0.tar.gz \ + && tar -zxf protobuf-2.5.0.tar.gz \ +# && wget https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/bigdata/Patch/protoc.zip \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protoc.zip \ + && unzip protoc.zip \ + && cp ./protoc/protoc.patch ./protobuf-2.5.0/src/google/protobuf/stubs/ \ + && cd protobuf-2.5.0/src/google/protobuf/stubs/ \ + && patch -p1 < protoc.patch \ + && cd ../../../../ \ + && ./autogen.sh && ./configure CFLAGS='-fsigned-char' && make -j8 && make install \ + && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc \ + && cd ../ \ && tar -zxf hbase-2.1.0-src.tar.gz \ && cd hbase-2.1.0 \ && sed -i '3676 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ diff --git a/BigData/CentOS8.1.1911/apache/Dockerfile-hive-3.0.0 b/BigData/CentOS8.1.1911/apache/Dockerfile-hive-3.0.0 index d74c137..1eecd88 100644 --- a/BigData/CentOS8.1.1911/apache/Dockerfile-hive-3.0.0 +++ b/BigData/CentOS8.1.1911/apache/Dockerfile-hive-3.0.0 @@ -1,5 +1,10 @@ FROM arm64v8/centos:1911 - +WORKDIR /etc/yum.repos.d/ +RUN sed -i "s/enabled=0/enabled=1/g" *repo \ + && mv CentOS-Media.repo CentOS-Media.repo.bak \ + && yum clean all \ + && yum makecache \ + && yum install -y patch unzip diffutils WORKDIR /root/ #RUN cp -r /etc/yum.repos.d /etc/yum.repos.d-bak \ # && rm /etc/yum.repos.d/* -rf \ @@ -31,11 +36,24 @@ ENV JAVA_HOME=/opt/tools/installed/jdk8u252-b09 ENV PATH=$JAVA_HOME/bin:$PATH ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.5.4 ENV PATH=$MAVEN_HOME/bin:$PATH -RUN yum install -y openssl-devel zlib-devel automake libtool cmake protobuf protobuf-devel \ - && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/bin/protoc \ +RUN yum install -y openssl-devel zlib-devel automake libtool cmake \ && cd /opt/ \ # && wget https://github.com/apache/hive/archive/rel/release-3.0.0.tar.gz \ && wget http://archive-server:50050/cache-data/BigData/Apache/hive/release-3.0.0.tar.gz \ + && yum install -y patch unzip \ +# && wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protobuf-2.5.0.tar.gz \ + && tar -zxf protobuf-2.5.0.tar.gz \ +# && wget https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/bigdata/Patch/protoc.zip \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protoc.zip \ + && unzip protoc.zip \ + && cp ./protoc/protoc.patch ./protobuf-2.5.0/src/google/protobuf/stubs/ \ + && cd protobuf-2.5.0/src/google/protobuf/stubs/ \ + && patch -p1 < protoc.patch \ + && cd ../../../../ \ + && ./autogen.sh && ./configure CFLAGS='-fsigned-char' && make -j8 && make install \ + && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc \ + && cd ../ \ && mv release-3.0.0.tar.gz hive-rel-release-3.0.0.tar.gz \ && tar -zxf hive-rel-release-3.0.0.tar.gz \ && cd hive-rel-release-3.0.0 \ diff --git a/BigData/CentOS8.1.1911/apache/Dockerfile-spark-2.3.0 b/BigData/CentOS8.1.1911/apache/Dockerfile-spark-2.3.0 index 0a33310..ac921a9 100644 --- a/BigData/CentOS8.1.1911/apache/Dockerfile-spark-2.3.0 +++ b/BigData/CentOS8.1.1911/apache/Dockerfile-spark-2.3.0 @@ -1,5 +1,10 @@ FROM arm64v8/centos:1911 - +WORKDIR /etc/yum.repos.d/ +RUN sed -i "s/enabled=0/enabled=1/g" *repo \ + && mv CentOS-Media.repo CentOS-Media.repo.bak \ + && yum clean all \ + && yum makecache \ + && yum install -y patch unzip diffutils WORKDIR /root/ #RUN cp -r /etc/yum.repos.d /etc/yum.repos.d-bak \ # && rm /etc/yum.repos.d/* -rf \ diff --git a/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 b/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 index f3c3c58..71f2cfb 100644 --- a/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 +++ b/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 @@ -1,5 +1,9 @@ FROM arm64v8/centos:1911 - +WORKDIR /etc/yum.repos.d/ +RUN sed -i "s/enabled=0/enabled=1/g" *repo \ + && mv CentOS-Media.repo CentOS-Media.repo.bak \ + && yum clean all \ + && yum makecache WORKDIR /root/ RUN yum install -y gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ && yum install vim wget -y automake libstdc++-static git glibv-static ant svn automake libtool autoconf cppunit-devel -- Gitee From 1181053d3c47ab1e0bbdd0a53e3286abfb74d089 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 19:23:58 +0800 Subject: [PATCH 2/2] match centos8 --- BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 | 6 +++++- BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 b/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 index 222424c..aab8e35 100644 --- a/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 +++ b/BigData/CentOS8.1.1911/apache/Dockerfile-hadoop-3.1.1 @@ -12,7 +12,7 @@ WORKDIR /root/ # && yum clean all \ # && yum makecache \ # && yum downgrade -y krb5-libs openssl-libs krb5-libs libstdc++ glibc glibc-devel glibc-common \ -RUN yum update -y +#RUN yum update -y RUN yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen \ && yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ && mv /usr/bin/gcc /usr/bin/gcc-impl \ @@ -77,6 +77,10 @@ RUN cd /opt/ \ && tar -zxvf hadoop-3.1.1-src.tar.gz \ && cd hadoop-3.1.1-src \ && sed -i '54 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ + && cd hadoop-tools/hadoop-pipes/src/ \ + && wget http://archive-server:50050/cache-data/BigData/Apache/hadoop/hadoop-3.1.1-centos8.patch \ + && patch < hadoop-3.1.1-centos8.patch \ + && cd - \ && mvn package -DskipTests -Pdist,native -Dtar -Dsnappy.lib=/usr/local/lib64 -Dbundle.snappy -Dmaven.javadoc.skip=true \ && mkdir -p /root/output \ && cp hadoop-dist/target/hadoop-3.1.1.tar.gz /root/output diff --git a/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 b/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 index 71f2cfb..9387e34 100644 --- a/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 +++ b/BigData/CentOS8.1.1911/apache/Dockerfile-zookeeper-3.4.6 @@ -41,5 +41,6 @@ RUN wget http://archive-server:50050/cache-data/BigData/Apache/zookeeper/release && sed -i 's/repo1\.maven\.org\/maven2/mirrors\.huaweicloud\.com\/repository\/maven/' ivysettings.xml \ && sed -i 's/http:\/\/repo2/https:\/\/repo1/' build.xml \ && ant package tar \ + || ant package tar \ && mkdir -p /root/output \ && cp ./build/zookeeper-3.4.6.tar.gz /root/output -- Gitee