diff --git a/client-mr/mr-shaded/pom.xml b/client-mr/mr-shaded/pom.xml index 2ffa40e1aad..0f9052f1f56 100644 --- a/client-mr/mr-shaded/pom.xml +++ b/client-mr/mr-shaded/pom.xml @@ -64,8 +64,8 @@ ${shading.prefix}.org.scala-lang - org.lz4 - ${shading.prefix}.org.lz4 + ${lz4-java.group} + ${shading.prefix}.${lz4-java.group} org.roaringbitmap @@ -81,7 +81,7 @@ io.netty:* org.apache.commons:commons-lang3 org.scala-lang:scala-library - org.lz4:lz4-java + ${lz4-java.group}:lz4-java com.github.luben:zstd-jni org.roaringbitmap:RoaringBitmap diff --git a/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE b/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE index ec665dcc837..7435dd2e5c3 100644 --- a/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE +++ b/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE @@ -208,6 +208,7 @@ This project bundles the following dependencies under the Apache License 2.0 (ht Apache License 2.0 -------------------------------------- +at.yawk.lz4:lz4-java com.google.guava:failureaccess com.google.guava:guava io.netty:netty @@ -240,7 +241,6 @@ io.netty:netty-transport-rxtx io.netty:netty-transport-sctp io.netty:netty-transport-udt org.apache.commons:commons-lang3 -org.lz4:lz4-java org.roaringbitmap:RoaringBitmap org.scala-lang:scala-library diff --git a/client-tez/tez-shaded/pom.xml b/client-tez/tez-shaded/pom.xml index e8060d95a81..8cc32071c91 100644 --- a/client-tez/tez-shaded/pom.xml +++ b/client-tez/tez-shaded/pom.xml @@ -94,7 +94,7 @@ org.roaringbitmap:RoaringBitmap org.scala-lang:scala-library org.scala-lang:scala-reflect - org.lz4:lz4-java + at.yawk.lz4:lz4-java io.dropwizard.metrics:metrics-core com.codahale.metrics:metrics-core com.github.luben:zstd-jni diff --git a/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE b/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE index ec665dcc837..7435dd2e5c3 100644 --- a/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE +++ b/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE @@ -208,6 +208,7 @@ This project bundles the following dependencies under the Apache License 2.0 (ht Apache License 2.0 -------------------------------------- +at.yawk.lz4:lz4-java com.google.guava:failureaccess com.google.guava:guava io.netty:netty @@ -240,7 +241,6 @@ io.netty:netty-transport-rxtx io.netty:netty-transport-sctp io.netty:netty-transport-udt org.apache.commons:commons-lang3 -org.lz4:lz4-java org.roaringbitmap:RoaringBitmap org.scala-lang:scala-library diff --git a/client/pom.xml b/client/pom.xml index 4b69dfedcc5..8f227ab8503 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -51,7 +51,7 @@ guava - org.lz4 + ${lz4-java.group} lz4-java diff --git a/dev/deps/dependencies-client-flink-1.16 b/dev/deps/dependencies-client-flink-1.16 index ab6caa152e4..38b2969fdae 100644 --- a/dev/deps/dependencies-client-flink-1.16 +++ b/dev/deps/dependencies-client-flink-1.16 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/dev/deps/dependencies-client-flink-1.17 b/dev/deps/dependencies-client-flink-1.17 index ab6caa152e4..38b2969fdae 100644 --- a/dev/deps/dependencies-client-flink-1.17 +++ b/dev/deps/dependencies-client-flink-1.17 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/dev/deps/dependencies-client-flink-1.18 b/dev/deps/dependencies-client-flink-1.18 index ab6caa152e4..38b2969fdae 100644 --- a/dev/deps/dependencies-client-flink-1.18 +++ b/dev/deps/dependencies-client-flink-1.18 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/dev/deps/dependencies-client-flink-1.19 b/dev/deps/dependencies-client-flink-1.19 index ab6caa152e4..38b2969fdae 100644 --- a/dev/deps/dependencies-client-flink-1.19 +++ b/dev/deps/dependencies-client-flink-1.19 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/dev/deps/dependencies-client-flink-1.20 b/dev/deps/dependencies-client-flink-1.20 index ab6caa152e4..38b2969fdae 100644 --- a/dev/deps/dependencies-client-flink-1.20 +++ b/dev/deps/dependencies-client-flink-1.20 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/dev/deps/dependencies-client-flink-2.0 b/dev/deps/dependencies-client-flink-2.0 index 031b3821ef2..80d1824ce88 100644 --- a/dev/deps/dependencies-client-flink-2.0 +++ b/dev/deps/dependencies-client-flink-2.0 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar diff --git a/dev/deps/dependencies-client-flink-2.1 b/dev/deps/dependencies-client-flink-2.1 index 031b3821ef2..80d1824ce88 100644 --- a/dev/deps/dependencies-client-flink-2.1 +++ b/dev/deps/dependencies-client-flink-2.1 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar diff --git a/dev/deps/dependencies-client-flink-2.2 b/dev/deps/dependencies-client-flink-2.2 index 031b3821ef2..80d1824ce88 100644 --- a/dev/deps/dependencies-client-flink-2.2 +++ b/dev/deps/dependencies-client-flink-2.2 @@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar jsr305/1.3.9//jsr305-1.3.9.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr index 35d26cec759..4b14da97616 100644 --- a/dev/deps/dependencies-client-mr +++ b/dev/deps/dependencies-client-mr @@ -134,7 +134,7 @@ kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar kotlin-stdlib-common/1.4.10//kotlin-stdlib-common-1.4.10.jar kotlin-stdlib/1.4.10//kotlin-stdlib-1.4.10.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/dev/deps/dependencies-client-tez b/dev/deps/dependencies-client-tez index 21edcd5e2ad..585baac8ceb 100644 --- a/dev/deps/dependencies-client-tez +++ b/dev/deps/dependencies-client-tez @@ -107,7 +107,7 @@ kerby-util/1.0.1//kerby-util-1.0.1.jar kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar log4j/1.2.17//log4j-1.2.17.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server index a526f0a4ccc..f7fa3285afb 100644 --- a/dev/deps/dependencies-server +++ b/dev/deps/dependencies-server @@ -79,7 +79,7 @@ log4j-1.2-api/2.24.3//log4j-1.2-api-2.24.3.jar log4j-api/2.24.3//log4j-api-2.24.3.jar log4j-core/2.24.3//log4j-core-2.24.3.jar log4j-slf4j-impl/2.24.3//log4j-slf4j-impl-2.24.3.jar -lz4-java/1.8.0//lz4-java-1.8.0.jar +lz4-java/1.10.1//lz4-java-1.10.1.jar maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar metrics-core/4.2.25//metrics-core-4.2.25.jar metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar diff --git a/pom.xml b/pom.xml index cd87d0d49bd..f5033a55255 100644 --- a/pom.xml +++ b/pom.xml @@ -91,7 +91,8 @@ 1.8 2.24.3 3.4.4 - 1.8.0 + at.yawk.lz4 + 1.10.1 4.11.0 1.17.14 4.1.118.Final @@ -441,7 +442,7 @@ ${leveldb.version} - org.lz4 + ${lz4-java.group} lz4-java ${lz4-java.version} @@ -1424,6 +1425,7 @@ tests/spark-it + org.lz4 1.4.0 2.11.12 2.11 @@ -1443,6 +1445,7 @@ tests/spark-it + org.lz4 1.7.1 2.12.10 2.12 @@ -1463,6 +1466,7 @@ tests/spark-it + org.lz4 1.7.1 2.12.10 2.12 @@ -1483,6 +1487,7 @@ tests/spark-it + org.lz4 1.7.1 2.12.15 2.12 @@ -1502,6 +1507,7 @@ tests/spark-it + org.lz4 1.8.0 2.12.15 2.12 @@ -1521,6 +1527,7 @@ tests/spark-it + org.lz4 1.8.0 2.12.17 2.12 @@ -1540,6 +1547,7 @@ tests/spark-it + org.lz4 1.8.0 2.12.18 2.12 @@ -1559,6 +1567,7 @@ tests/spark-it + org.lz4 1.8.0 2.13.16 2.13 diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index 753bf1eb4fd..2cadc5512e3 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -38,7 +38,8 @@ import CelebornCommonSettings._ object Dependencies { val zstdJniVersion = sparkClientProjects.map(_.zstdJniVersion).getOrElse("1.5.7-1") - val lz4JavaVersion = sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.8.0") + val lz4JavaGroup = sparkClientProjects.map(_.lz4JavaGroup).getOrElse("at.yawk.lz4") + val lz4JavaVersion = sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.10.1") // Dependent library versions val apLoaderVersion = "4.0-10" @@ -152,7 +153,7 @@ object Dependencies { val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version val disruptor = "com.lmax" % "disruptor" % disruptorVersion - val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion + val lz4Java = lz4JavaGroup % "lz4-java" % lz4JavaVersion val protobufJava = "com.google.protobuf" % "protobuf-java" % protoVersion val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion val ratisCommon = "org.apache.ratis" % "ratis-common" % ratisVersion @@ -946,6 +947,7 @@ trait SparkClientProjects { val sparkClientShadedProjectPath: String val sparkClientShadedProjectName: String + val lz4JavaGroup: String = "org.lz4" val lz4JavaVersion: String val sparkProjectScalaVersion: String val sparkVersion: String