packages/spark
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix build on riscv64

Browse Source
This commit is contained in:
Dingli Zhang 2024-08-01 03:57:48 +00:00
parent 5f3df7caea
commit 7ea0551324
11 changed files with 316 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
0002-Upgrade-os-maven-plugin-to-1.7.1.patch Normal file
View File

@ -0,0 +1,22 @@
From 958468fc5154399f1a256097502b21a8f256b2f3 Mon Sep 17 00:00:00 2001
From: Dingli Zhang <dingli@iscas.ac.cn>
Date: Mon, 1 Jul 2024 05:21:50 +0000
Subject: [PATCH] Upgrade os-maven-plugin to 1.7.1
diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
index c78c5445..57515a8f 100644
--- a/connector/connect/common/pom.xml
+++ b/connector/connect/common/pom.xml
@@ -116,7 +116,7 @@
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
- <version>1.6.2</version>
+ <version>1.7.1</version>
</extension>
</extensions>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
--
2.41.0

56
1000-Added-support-for-building-the-riscv64-protoc-binari.patch Normal file
View File

@ -0,0 +1,56 @@
From 8b626946fcabc659a41de3b15cbe6f5d45bd596e Mon Sep 17 00:00:00 2001
From: Dingli Zhang <dingli@iscas.ac.cn>
Date: Tue, 7 May 2024 13:56:04 +0800
Subject: [PATCH] Added support for building the riscv64 protoc binaries with
Kokoro
diff --git a/kokoro/release/protoc/linux/build.sh b/kokoro/release/protoc/linux/build.sh
index 44a95aab3..13be7b2ed 100755
--- a/kokoro/release/protoc/linux/build.sh
+++ b/kokoro/release/protoc/linux/build.sh
@@ -22,6 +22,9 @@ protoc-artifacts/build-protoc.sh linux aarch_64 protoc
sudo apt install -y g++-powerpc64le-linux-gnu
protoc-artifacts/build-protoc.sh linux ppcle_64 protoc
+sudo apt install -y g++-riscv64-linux-gnu
+protoc-artifacts/build-protoc.sh linux riscv64 protoc
+
sudo apt install -y g++-s390x-linux-gnu
protoc-artifacts/build-protoc.sh linux s390_64 protoc
diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh
index 460284262..20ca05739 100755
--- a/protoc-artifacts/build-protoc.sh
+++ b/protoc-artifacts/build-protoc.sh
@@ -93,6 +93,8 @@ checkArch ()
assertEq $format "elf64-x86-64" $LINENO
elif [[ "$ARCH" == aarch_64 ]]; then
assertEq $format "elf64-little" $LINENO
+ elif [[ "$ARCH" == riscv64 ]]; then
+ assertEq $format "elf64-littleriscv" $LINENO
elif [[ "$ARCH" == s390_64 ]]; then
if [[ $host_machine == s390x ]];then
assertEq $format "elf64-s390" $LINENO
@@ -162,6 +164,9 @@ checkDependencies ()
elif [[ "$ARCH" == aarch_64 ]]; then
dump_cmd='objdump -p '"$1"' | grep NEEDED'
white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1"
+ elif [[ "$ARCH" == riscv64 ]]; then
+ dump_cmd='objdump -p '"$1"' | grep NEEDED'
+ white_list="libz\.so\.1\|libm\.so\.6\|libc\.so\.6\|ld-linux-riscv64-lp64d\.so\.1"
fi
elif [[ "$OS" == osx ]]; then
dump_cmd='otool -L '"$1"' | fgrep dylib'
@@ -226,6 +231,8 @@ elif [[ "$(uname)" == Linux* ]]; then
elif [[ "$ARCH" == ppcle_64 ]]; then
CXXFLAGS="$CXXFLAGS -m64"
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=powerpc64le-linux-gnu"
+ elif [[ "$ARCH" == riscv64 ]]; then
+ CONFIGURE_ARGS="$CONFIGURE_ARGS --host=riscv64-linux-gnu"
elif [[ "$ARCH" == s390_64 ]]; then
CXXFLAGS="$CXXFLAGS -m64"
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=s390x-linux-gnu"
--
2.41.0

23
1001-Add-protoc-java-support-for-riscv64.patch Normal file
View File

@ -0,0 +1,23 @@
From 2c52801962444baa444401b8fafd09bd6cfab92b Mon Sep 17 00:00:00 2001
From: Dingli Zhang <dingli@iscas.ac.cn>
Date: Thu, 9 May 2024 00:02:35 +0800
Subject: [PATCH] Add support for riscv64
---
src/main/java/com/github/os72/protocjar/PlatformDetector.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/main/java/com/github/os72/protocjar/PlatformDetector.java b/src/main/java/com/github/os72/protocjar/PlatformDetector.java
index 7a99348..b8f08e5 100644
--- a/src/main/java/com/github/os72/protocjar/PlatformDetector.java
+++ b/src/main/java/com/github/os72/protocjar/PlatformDetector.java
@@ -211,6 +211,9 @@ private static String normalizeArch(String value) {
if ("ppc64le".equals(value)) {
return "ppcle_64";
}
+ if ("riscv64".equals(value)) {
+ return "riscv64";
+ }
if ("s390".equals(value)) {
return "s390_32";
}

147
1002-Added-support-for-building-the-riscv64-protoc-gen-gr.patch Normal file
View File

@ -0,0 +1,147 @@
From 11c8f77c101385119e34d0961e4203c6eb27c52d Mon Sep 17 00:00:00 2001
From: Dingli Zhang <dingli@iscas.ac.cn>
Date: Fri, 5 Jul 2024 03:48:20 +0000
Subject: [PATCH] Added support for building the riscv64 protoc-gen-grpc-java
binaries with Kokoro
diff --git a/build.gradle b/build.gradle
index ed4baab82..c5bed1026 100644
--- a/build.gradle
+++ b/build.gradle
@@ -23,6 +23,8 @@ subprojects {
version = "1.56.0" // CURRENT_GRPC_VERSION
repositories {
+ maven { url 'file://@HOME@/.m2/repository'}
+ maven { url "https://mirrors.huaweicloud.com/repository/maven/"}
maven { // The google mirror is less flaky than mavenCentral()
url "https://maven-central.storage-download.googleapis.com/maven2/" }
mavenCentral()
diff --git a/buildscripts/kokoro/linux_artifacts.sh b/buildscripts/kokoro/linux_artifacts.sh
index c80d3436f..49a15a457 100755
--- a/buildscripts/kokoro/linux_artifacts.sh
+++ b/buildscripts/kokoro/linux_artifacts.sh
@@ -56,6 +56,8 @@ cp -r "$LOCAL_MVN_TEMP"/* "$MVN_ARTIFACT_DIR"/
SKIP_TESTS=true ARCH=aarch_64 /grpc-java/buildscripts/kokoro/unix.sh
"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-multiarch env \
SKIP_TESTS=true ARCH=ppcle_64 /grpc-java/buildscripts/kokoro/unix.sh
+"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-multiarch env \
+ SKIP_TESTS=true ARCH=risv64 /grpc-java/buildscripts/kokoro/unix.sh
# Use a newer GCC version. GCC 7 in multiarch has a bug:
# internal compiler error: output_operand: invalid %-code
"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-ubuntu2004 env \
diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh
index 1e6af6cfc..048aef9e8 100755
--- a/buildscripts/kokoro/unix.sh
+++ b/buildscripts/kokoro/unix.sh
@@ -48,7 +48,7 @@ GRADLE_FLAGS+=" -Pcheckstyle.ignoreFailures=false"
GRADLE_FLAGS+=" -PfailOnWarnings=true"
GRADLE_FLAGS+=" -PerrorProne=true"
GRADLE_FLAGS+=" -PskipAndroid=true"
-GRADLE_FLAGS+=" -Dorg.gradle.parallel=true"
+GRADLE_FLAGS+=" -Dorg.gradle.parallel=false"
export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
# Make protobuf discoverable by :grpc-compiler
@@ -90,7 +90,7 @@ fi
LOCAL_MVN_TEMP=$(mktemp -d)
# Note that this disables parallel=true from GRADLE_FLAGS
if [[ -z "${ALL_ARTIFACTS:-}" ]]; then
- if [[ "$ARCH" = "aarch_64" || "$ARCH" = "ppcle_64" || "$ARCH" = "s390_64" ]]; then
+ if [[ $ARCH == "aarch_64" || $ARCH == "riscv64" ]]; then
GRADLE_FLAGS+=" -x grpc-compiler:generateTestProto -x grpc-compiler:generateTestLiteProto"
GRADLE_FLAGS+=" -x grpc-compiler:testGolden -x grpc-compiler:testLiteGolden"
GRADLE_FLAGS+=" -x grpc-compiler:testDeprecatedGolden -x grpc-compiler:testDeprecatedLiteGolden"
diff --git a/buildscripts/kokoro/upload_artifacts.sh b/buildscripts/kokoro/upload_artifacts.sh
index 39e27eff5..bc47d4b50 100644
--- a/buildscripts/kokoro/upload_artifacts.sh
+++ b/buildscripts/kokoro/upload_artifacts.sh
@@ -34,6 +34,9 @@ LOCAL_OTHER_ARTIFACTS="$KOKORO_GFILE_DIR"/github/grpc-java/artifacts/
# for linux aarch64 platform
[[ "$(find "$LOCAL_MVN_ARTIFACTS" -type f -iname 'protoc-gen-grpc-java-*-linux-aarch_64.exe' | wc -l)" != '0' ]]
+# for linux riscv64 platform
+[[ "$(find "$LOCAL_MVN_ARTIFACTS" -type f -iname 'protoc-gen-grpc-java-*-linux-riscv64.exe' | wc -l)" != '0' ]]
+
# for linux ppc64le platform
[[ "$(find "$LOCAL_MVN_ARTIFACTS" -type f -iname 'protoc-gen-grpc-java-*-linux-ppcle_64.exe' | wc -l)" != '0' ]]
diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh
index 3d02a72f4..0427d47e1 100755
--- a/buildscripts/make_dependencies.sh
+++ b/buildscripts/make_dependencies.sh
@@ -7,8 +7,8 @@ PROTOBUF_VERSION=21.7
# ARCH is x86_64 bit unless otherwise specified.
ARCH="${ARCH:-x86_64}"
-DOWNLOAD_DIR=/tmp/source
-INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH"
+DOWNLOAD_DIR=/var/tmp/source
+INSTALL_DIR="/var/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH"
mkdir -p $DOWNLOAD_DIR
# Start with a sane default
@@ -36,6 +36,8 @@ else
--prefix="$INSTALL_DIR"
elif [[ "$ARCH" == aarch* ]]; then
./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR"
+ elif [[ "$ARCH" == riscv* ]]; then
+ ./configure --disable-shared --host=riscv64-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == ppc* ]]; then
./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == s390* ]]; then
diff --git a/compiler/build.gradle b/compiler/build.gradle
index 12798208c..7960642cd 100644
--- a/compiler/build.gradle
+++ b/compiler/build.gradle
@@ -58,6 +58,7 @@ model {
cppCompiler.executable = 'aarch64-linux-gnu-g++'
linker.executable = 'aarch64-linux-gnu-g++'
}
+ target("riscv64")
target("s390_64") {
cppCompiler.executable = 's390x-linux-gnu-g++'
linker.executable = 's390x-linux-gnu-g++'
@@ -73,6 +74,7 @@ model {
x86_64 { architecture "x86_64" }
ppcle_64 { architecture "ppcle_64" }
aarch_64 { architecture "aarch_64" }
+ riscv64 { architecture "riscv64" }
s390_64 { architecture "s390_64" }
loongarch_64 { architecture "loongarch_64" }
}
@@ -84,6 +86,7 @@ model {
'x86_64',
'ppcle_64',
'aarch_64',
+ 'riscv64',
's390_64',
'loongarch_64'
]) {
diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh
index a5f33a35b..ebe9dc4a1 100755
--- a/compiler/check-artifact.sh
+++ b/compiler/check-artifact.sh
@@ -61,6 +61,8 @@ checkArch ()
assertEq "$format" "elf64-x86-64" $LINENO
elif [[ "$ARCH" == aarch_64 ]]; then
assertEq "$format" "elf64-little" $LINENO
+ elif [[ "$ARCH" == riscv64 ]]; then
+ assertEq $format "elf64-littleriscv" $LINENO
elif [[ "$ARCH" == loongarch_64 ]]; then
echo $format
assertEq "$format" "elf64-loongarch" $LINENO
@@ -119,6 +121,9 @@ checkDependencies ()
elif [[ "$ARCH" == aarch_64 ]]; then
dump_cmd='aarch64-linux-gnu-objdump -x '"$1"' |grep "NEEDED"'
white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1"
+ elif [[ "$ARCH" == riscv64 ]]; then
+ dump_cmd='objdump -p '"$1"' | grep NEEDED'
+ white_list="libm\.so\.6\|libc\.so\.6\|ld-linux-riscv64-lp64d\.so\.1"
elif [[ "$ARCH" == loongarch_64 ]]; then
dump_cmd='objdump -x '"$1"' | grep NEEDED'
white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld\.so\.1"
--
2.41.0

BIN
gradle-7.6-bin.zip Normal file
View File

Binary file not shown.

BIN
protobuf-all-21.7.tar.gz Normal file
View File

Binary file not shown.

71
spark.spec
View File

@ -4,28 +4,88 @@
Summary: A unified analytics engine for large-scale data processing.
Name: spark
Version: 3.5.0
Release: 2
Release: 3
License: Apache 2.0
URL: http://spark.apache.org/
Source0: https://github.com/apache/spark/archive/v%{version}.tar.gz
Source1: https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.7.tar.gz
Source2: https://github.com/os72/protoc-jar/archive/refs/tags/v3.11.4.tar.gz
Source3: https://github.com/os72/protoc-jar-maven-plugin/archive/refs/tags/v3.11.4.zip
Source4: https://services.gradle.org/distributions/gradle-7.6-bin.zip
Source5: https://github.com/google/protobuf/releases/download/v21.7/protobuf-all-21.7.tar.gz
Source6: https://github.com/grpc/grpc-java/archive/refs/tags/v1.56.0.tar.gz
Patch0001: 0001-change-mvn-scalafmt.patch
Patch0002: 0002-Upgrade-os-maven-plugin-to-1.7.1.patch
%ifarch riscv64
BuildRequires: protobuf-devel protobuf-compiler
BuildRequires: autoconf automake libtool pkgconfig zlib-devel libstdc++-static gcc-c++
Patch1000: 1000-Added-support-for-building-the-riscv64-protoc-binari.patch
Patch1001: 1001-Add-protoc-java-support-for-riscv64.patch
Patch1002: 1002-Added-support-for-building-the-riscv64-protoc-gen-gr.patch
%endif
BuildRequires: java-1.8.0-openjdk-devel git maven
Requires: java-1.8.0-openjdk
ExclusiveArch: x86_64 aarch64 loongarch64
ExclusiveArch: x86_64 aarch64 loongarch64 riscv64
%description
Apache Spark achieves high performance for both batch and streaming data, using a state-of-the-art DAG scheduler, a query optimizer, and a physical execution engine.
%prep
%ifarch riscv64
mkdir -p ${HOME}/%{name}-prep_dir
export PROTOC_VERSION="3."$(protoc --version | awk '{print $NF}')
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=${PROTOC_VERSION} -Dclassifier=linux-riscv64 -Dpackaging=exe -Dfile=/usr/bin/protoc
# protoc
tar -mxf %{SOURCE1} -C ${HOME}/%{name}-prep_dir
pushd ${HOME}/%{name}-prep_dir/protobuf-3.21.7
%patch1000 -p1
./autogen.sh
./protoc-artifacts/build-protoc.sh linux riscv64 protoc
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=3.21.7 -Dclassifier=linux-riscv64 -Dpackaging=exe -Dfile=protoc-artifacts/target/linux/riscv64/protoc.exe
popd
# protoc-jar
tar -mxf %{SOURCE2} -C ${HOME}/%{name}-prep_dir
pushd ${HOME}/%{name}-prep_dir/protoc-jar-3.11.4
%patch1001 -p1
mvn clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
popd
# protoc-jar-maven-plugin
unzip %{SOURCE3} -d ${HOME}/%{name}-prep_dir
pushd ${HOME}/%{name}-prep_dir/protoc-jar-maven-plugin-3.11.4
mvn clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
popd
# prepare gradle and protobuf for build protoc-gen-grpc-java
mkdir -p %{_tmppath}/source
cp %{SOURCE4} %{_tmppath}/source
tar xzf %{SOURCE5} -C %{_tmppath}/source
# protoc-gen-grpc-java
tar -mxf %{SOURCE6} -C ${HOME}/%{name}-prep_dir
pushd ${HOME}/%{name}-prep_dir/grpc-java-1.56.0
%patch1002 -p1
sed -i "s,@HOME@,${HOME},g" build.gradle
sed -i 's|https\\://services.gradle.org/distributions|file://%{_tmppath}/source|g' gradle/wrapper/gradle-wrapper.properties
SKIP_TESTS=true ARCH=riscv64 ./buildscripts/kokoro/unix.sh
mvn install:install-file -DgroupId=io.grpc -DartifactId=protoc-gen-grpc-java -Dversion=1.56.0 -Dclassifier=linux-riscv64 -Dpackaging=exe -Dfile=mvn-artifacts/io/grpc/protoc-gen-grpc-java/1.56.0/protoc-gen-grpc-java-1.56.0-linux-riscv64.exe
popd
%endif
%setup -q
%patch0001 -p1
%patch0002 -p1
%ifarch riscv64
sed -i -e 's/protoVersion = "3.23.4/protoVersion = "'${PROTOC_VERSION}/'' project/SparkBuild.scala
sed -i -e 's/<protobuf.version>3.23.4/<protobuf.version>'${PROTOC_VERSION}/'' pom.xml
%endif
%build
%ifarch riscv64
export MAVEN_OPTS="-Xms2048M -Xmx8000M"
%endif
./dev/make-distribution.sh --mvn mvn --name spark-3.5.0 --tgz -DskipTests -Dmaven.test.skip=true
%install
@ -37,6 +97,11 @@ cp -rf ../%{name}-%{version} %{buildroot}/opt/apache-%{name}-%{version}
%changelog
* Mon Jul 1 2024 Dingli Zhang <dingli@iscas.ac.cn> - 3.5.0-3
- Add riscv64 to ExclusiveArch
- Fix build on riscv64
- Upgrade os-maven-plugin to 1.7.1
* Tue May 21 2024 Pengda Dou <doupengda@loongson.cn> - 3.5.0-2
- add loongarch64 to ExclusiveArch

BIN
v1.56.0.tar.gz Normal file
View File

Binary file not shown.

BIN
v3.11.4.tar.gz Normal file
View File

Binary file not shown.

BIN
v3.11.4.zip Normal file
View File

Binary file not shown.

BIN
v3.21.7.tar.gz Normal file
View File

Binary file not shown.