Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions precompiled/hadoop/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,18 @@ RUN <<EOF
# Create snapshot of the source code including custom patches
tar -czf /stackable/protobuf-${PROTOBUF_VERSION}-src.tar.gz .

./autogen.sh
./configure --prefix=/opt/protobuf
make "-j$(nproc)"
make install
if [ "$PROTOBUF_VERSION" == "3.7.1" ] || [ "$PROTOBUF_VERSION" == "3.21.12" ]; then
./autogen.sh
./configure --prefix=/opt/protobuf
make "-j$(nproc)"
make install
else
# protobuf > 3.21 bundles abseil-cpp and utf8_range as git submodules
git submodule update --init --recursive
cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
cmake --build build --parallel "$(nproc)"
cmake --install build --prefix /opt/protobuf
fi
(cd .. && rm -r ${PROTOBUF_VERSION})
EOF

Expand Down
12 changes: 9 additions & 3 deletions precompiled/hadoop/boil-config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,14 @@ java-devel = "11"
[versions."3.3.6".build-arguments]
protobuf-version = "3.7.1"

[versions."3.4.2".local-images]
[versions."3.4.3".local-images]
java-devel = "11"

[versions."3.4.2".build-arguments]
protobuf-version = "3.7.1"
[versions."3.4.3".build-arguments]
protobuf-version = "3.21.12"

[versions."3.5.0".local-images]
java-devel = "17"

[versions."3.5.0".build-arguments]
protobuf-version = "3.25.5"

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
From c4dbb05b4f92f93c7e8f11d6a622b73f40f4664c Mon Sep 17 00:00:00 2001
From: xeniape <xenia.fischer@stackable.tech>
Date: Wed, 10 Sep 2025 14:18:38 +0200
From 7ad31a922a9fbcecd884b4bdf5c416f6b0ea539e Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 26 May 2026 15:40:40 +0200
Subject: YARN-11527-Update-node.js

---
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index b9eacd5ba3..70f64bf55c 100644
index 0813904f98a..f837b1f5201 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -234,7 +234,7 @@
<jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
@@ -236,7 +236,7 @@
<woodstox.version>5.4.0</woodstox.version>
<nimbus-jose-jwt.version>9.37.2</nimbus-jose-jwt.version>
<nimbus-jose-jwt.version>10.4</nimbus-jose-jwt.version>
<jcip-annotations.version>1.0-1</jcip-annotations.version>
- <nodejs.version>v12.22.1</nodejs.version>
+ <nodejs.version>v14.17.0</nodejs.version>
<yarnpkg.version>v1.22.5</yarnpkg.version>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From adc337817824ba29e7eb669c13730acdbb0b9630 Mon Sep 17 00:00:00 2001
From d1266f7390327e1882f2646ffe595a961b56b592 Mon Sep 17 00:00:00 2001
From: xeniape <xenia.fischer@stackable.tech>
Date: Wed, 10 Sep 2025 14:36:20 +0200
Subject: Allow-overriding-datanode-registration-addresses
Expand All @@ -11,7 +11,7 @@ Subject: Allow-overriding-datanode-registration-addresses
4 files changed, 135 insertions(+), 22 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f92a2ad565..25bcd438c7 100755
index 6375778e4ea..7d49fbcfaa3 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -152,6 +152,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
Expand All @@ -38,7 +38,7 @@ index f92a2ad565..25bcd438c7 100755
public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true;

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index ebd2fa992e..c56f254478 100644
index ebd2fa992e9..c56f254478b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -181,6 +181,8 @@ public class DatanodeManager {
Expand Down Expand Up @@ -110,7 +110,7 @@ index ebd2fa992e..c56f254478 100644
// it will be disallowed from registering.
if (!hostConfigManager.isIncluded(nodeReg)) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 21b92db307..5d3437239c 100644
index 21b92db3073..5d3437239ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -101,6 +101,11 @@ public class DNConf {
Expand Down Expand Up @@ -205,7 +205,7 @@ index 21b92db307..5d3437239c 100644
* Returns socket timeout
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 956f5bbe51..22ae127d98 100644
index 956f5bbe519..22ae127d98d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -135,6 +135,7 @@ import java.util.HashSet;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From ab9550bd7b71c16c381a105a22732f6e71f2dba6 Mon Sep 17 00:00:00 2001
From 8f29f6a32a0685577023398af3e7b2854ad47e96 Mon Sep 17 00:00:00 2001
From: xeniape <xenia.fischer@stackable.tech>
Date: Wed, 10 Sep 2025 14:39:20 +0200
Subject: Async-profiler-also-grab-itimer-events
Expand All @@ -8,7 +8,7 @@ Subject: Async-profiler-also-grab-itimer-events
1 file changed, 2 insertions(+)

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
index ce53274151..909892ff90 100644
index ce532741512..909892ff903 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.util.ProcessUtils;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From 2b131b13ad062695d5de8840be744900fa4a71c1 Mon Sep 17 00:00:00 2001
From a79df04a30b4477372e850a7628bd47dcb665955 Mon Sep 17 00:00:00 2001
From: xeniape <xenia.fischer@stackable.tech>
Date: Wed, 10 Sep 2025 14:40:41 +0200
Subject: HDFS-17378-Fix-missing-operationType-for-some-operat
Expand All @@ -8,7 +8,7 @@ Subject: HDFS-17378-Fix-missing-operationType-for-some-operat
1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index cfc7f24381..e9c5fc0da2 100644
index cfc7f24381f..e9c5fc0da25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2618,15 +2618,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From 18a413bfaec1acd447b070538be994ea9691899a Mon Sep 17 00:00:00 2001
From a1b7a73b7ebe8ad84ab3f417d2b475dd2b51c192 Mon Sep 17 00:00:00 2001
From: xeniape <xenia.fischer@stackable.tech>
Date: Wed, 10 Sep 2025 14:41:18 +0200
Subject: Update-CycloneDX-plugin
Expand All @@ -8,7 +8,7 @@ Subject: Update-CycloneDX-plugin
1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9a9a0de744..c7838e3674 100644
index d34fb58e22c..d30fd28be1f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -121,7 +121,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From e67476ea801e929feb3d89d023d915efbf82336f Mon Sep 17 00:00:00 2001
From 701aa813622cc8f78ac5c49ba9c4a9d638596657 Mon Sep 17 00:00:00 2001
From: xeniape <xenia.fischer@stackable.tech>
Date: Wed, 10 Sep 2025 14:41:46 +0200
Subject: HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-
Expand All @@ -14,10 +14,10 @@ Subject: HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-
create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 70f64bf55c..6d05a218d0 100644
index f837b1f5201..f8c7f2388f3 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1663,7 +1663,7 @@
@@ -1692,7 +1692,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
Expand All @@ -27,7 +27,7 @@ index 70f64bf55c..6d05a218d0 100644
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index c2e04623ad..5b12e4a4ef 100644
index 50d6d4576c5..42fa66976af 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -165,5 +165,31 @@
Expand Down Expand Up @@ -63,7 +63,7 @@ index c2e04623ad..5b12e4a4ef 100644
</dependencies>
</project>
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index ccd5d1ea25..30da259fd5 100644
index ccd5d1ea25c..30da259fd51 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -73,6 +73,7 @@ import java.util.List;
Expand Down Expand Up @@ -104,7 +104,7 @@ index ccd5d1ea25..30da259fd5 100644
int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION);
if (listVersion < 1 || listVersion > 2) {
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index baeb919937..176669ed15 100644
index baeb9199377..176669ed152 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -211,4 +211,19 @@ public final class Constants {
Expand All @@ -129,7 +129,7 @@ index baeb919937..176669ed15 100644
}
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
new file mode 100644
index 0000000000..5070f2a581
index 00000000000..5070f2a5816
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
@@ -0,0 +1,98 @@
Expand Down Expand Up @@ -232,7 +232,7 @@ index 0000000000..5070f2a581
+ }
+}
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
index bb5cbe5ec3..2167f68811 100644
index bb5cbe5ec32..2167f68811a 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
@@ -21,3 +21,6 @@ log4j.threshold=ALL
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
From 08a7206a29f212e1f0e3bd81e0cb0be7907907a4 Mon Sep 17 00:00:00 2001
From: Andrew Kenworthy <andrew.kenworthy@stackable.tech>
Date: Wed, 1 Oct 2025 17:28:41 +0200
Subject: Update nodejs to LTS version
From 8177d73d3dd9260f36c8bd349a0b6602e1a8fc2c Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 26 May 2026 15:43:29 +0200
Subject: YARN-11873 Update nodejs to LTS version

---
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2e21ff7394..a7d5faf760 100644
index f8c7f2388f3..4522eea9fe0 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -234,7 +234,7 @@
<jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
@@ -236,7 +236,7 @@
<woodstox.version>5.4.0</woodstox.version>
<nimbus-jose-jwt.version>9.37.4</nimbus-jose-jwt.version>
<nimbus-jose-jwt.version>10.4</nimbus-jose-jwt.version>
<jcip-annotations.version>1.0-1</jcip-annotations.version>
- <nodejs.version>v14.17.0</nodejs.version>
+ <nodejs.version>v22.20.0</nodejs.version>
<yarnpkg.version>v1.22.5</yarnpkg.version>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
From dff0783ae6c7026515a264bbf0fda137331a2864 Mon Sep 17 00:00:00 2001
From: Jim Halfpenny <jim.halfpenny@stackable.tech>
Date: Mon, 16 Mar 2026 09:18:46 +0000
Subject: HDFS-17891 fix for hostname resolution bug with datanodes

---
.../hadoop/hdfs/server/blockmanagement/HostSet.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
index d12e5fbae13..3da753f1cb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
@@ -18,10 +18,11 @@
package org.apache.hadoop.hdfs.server.blockmanagement;


-import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap;
import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap;
import org.apache.hadoop.thirdparty.com.google.common.collect.UnmodifiableIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;

import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -38,6 +39,8 @@ import java.util.Map;
* .getPort() || B.getPort() == 0.
*/
public class HostSet implements Iterable<InetSocketAddress> {
+ private static final Logger LOG = LoggerFactory.getLogger(HostSet.class);
+
// Host -> lists of ports
private final Multimap<InetAddress, Integer> addrs = HashMultimap.create();

@@ -72,7 +75,11 @@ public class HostSet implements Iterable<InetSocketAddress> {
}

void add(InetSocketAddress addr) {
- Preconditions.checkArgument(!addr.isUnresolved());
+ LOG.debug("Adding address to HostSet: {}", addr);
+ if (addr.isUnresolved()) {
+ LOG.warn("Unresolved address not added to HostSet: {}", addr);
+ return;
+ }
addrs.put(addr.getAddress(), addr.getPort());
}

Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
mirror = "https://github.com/stackabletech/hadoop.git"
base = "84e8b89ee2ebe6923691205b9e171badde7a495c"
base = "9d50c6884666e794e45102260a4017bb31802e1b"
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
From 151c1c7e4e47849a940c6c60896269f1c7bc11c8 Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 26 May 2026 15:40:40 +0200
Subject: YARN-11527-Update-node.js

---
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 4b9c22319cb..23657fb6e80 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -236,7 +236,7 @@
<woodstox.version>5.4.0</woodstox.version>
<nimbus-jose-jwt.version>10.4</nimbus-jose-jwt.version>
<jcip-annotations.version>1.0-1</jcip-annotations.version>
- <nodejs.version>v12.22.1</nodejs.version>
+ <nodejs.version>v14.17.0</nodejs.version>
<yarnpkg.version>v1.22.5</yarnpkg.version>
<apache-ant.version>1.10.13</apache-ant.version>
<jmh.version>1.20</jmh.version>
Loading
Loading