diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java index cc5270e1cdb..7b629f9d1dd 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java @@ -17,11 +17,12 @@ package org.apache.fluss.lake.lakestorage; +import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.plugin.PluginManager; -import org.apache.fluss.shaded.guava32.com.google.common.collect.Iterators; import javax.annotation.Nullable; +import java.util.Collections; import java.util.Iterator; import java.util.Objects; import java.util.ServiceLoader; @@ -32,35 +33,65 @@ */ public class LakeStoragePluginSetUp { + /** + * Finds the {@link LakeStoragePlugin} for the given datalake format, from the plugins directory + * or from the main classpath. + * + *

Providing it from both cannot work: {@code org.apache.fluss.} is parent-first, so the copy + * on the main classpath shadows the plugin and is loaded without the dependencies bundled next + * to it. That is rejected here rather than failing later with a {@link NoClassDefFoundError}. + */ public static LakeStoragePlugin fromDataLakeFormat( final String dataLakeFormat, @Nullable final PluginManager pluginManager) { - // now, load lake storage plugin - Iterator lakeStoragePluginIterator = - getAllLakeStoragePlugins(pluginManager); + LakeStoragePlugin fromPluginsDir = + findByIdentifier(loadFromPluginManager(pluginManager), dataLakeFormat); + LakeStoragePlugin fromClasspath = findByIdentifier(loadFromClasspath(), dataLakeFormat); - while (lakeStoragePluginIterator.hasNext()) { - LakeStoragePlugin lakeStoragePlugin = lakeStoragePluginIterator.next(); + if (fromPluginsDir != null && fromClasspath != null) { + throw new FlussRuntimeException( + String.format( + "Found two LakeStoragePlugin for datalake format '%s': one in the plugins " + + "directory and one on the main classpath (usually /lib). " + + "The copy on the main classpath shadows the one in the plugins " + + "directory, but is loaded without the dependencies bundled next to " + + "the plugin, such as Hadoop. This later fails with errors like " + + "'NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration'. " + + "Remove the fluss-lake-%s jar from /lib and keep it only " + + "in /plugins/%s/.", + dataLakeFormat, dataLakeFormat, dataLakeFormat)); + } + + LakeStoragePlugin lakeStoragePlugin = + fromPluginsDir != null ? fromPluginsDir : fromClasspath; + if (lakeStoragePlugin == null) { + throw new UnsupportedOperationException( + "No LakeStoragePlugin can be found for datalake format: " + dataLakeFormat); + } + return PluginLakeStorageWrapper.of(lakeStoragePlugin); + } + + @Nullable + private static LakeStoragePlugin findByIdentifier( + Iterator lakeStoragePlugins, String dataLakeFormat) { + while (lakeStoragePlugins.hasNext()) { + LakeStoragePlugin lakeStoragePlugin = lakeStoragePlugins.next(); if (Objects.equals(lakeStoragePlugin.identifier(), dataLakeFormat)) { - return PluginLakeStorageWrapper.of(lakeStoragePlugin); + return lakeStoragePlugin; } } - - // if come here, means we haven't found LakeStoragePlugin match the configured - // datalake, throw exception - throw new UnsupportedOperationException( - "No LakeStoragePlugin can be found for datalake format: " + dataLakeFormat); + return null; } - private static Iterator getAllLakeStoragePlugins( + private static Iterator loadFromPluginManager( @Nullable PluginManager pluginManager) { - final Iterator pluginIteratorSPI = - ServiceLoader.load( - LakeStoragePlugin.class, LakeStoragePlugin.class.getClassLoader()) - .iterator(); if (pluginManager == null) { - return pluginIteratorSPI; - } else { - return Iterators.concat(pluginManager.load(LakeStoragePlugin.class), pluginIteratorSPI); + return Collections.emptyIterator(); } + return pluginManager.load(LakeStoragePlugin.class); + } + + private static Iterator loadFromClasspath() { + return ServiceLoader.load(LakeStoragePlugin.class, LakeStoragePlugin.class.getClassLoader()) + .iterator(); } } diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java index b527bb6e483..25f97ccb80f 100644 --- a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.lakestorage; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.lake.source.LakeSource; @@ -98,6 +99,36 @@ void testWithPluginManager() throws Exception { .isInstanceOf(TestPaimonLakeCatalog.class); } + @Test + void testPluginFoundInBothPluginsDirAndClasspath() { + // same identifier from the plugin manager and from SPI, as when the jar is also in lib + final Map, Iterator> lakeStoragePlugins = new HashMap<>(); + lakeStoragePlugins.put( + LakeStoragePlugin.class, + Collections.singletonList(new TestingClasspathLakeStoragePlugin()).iterator()); + + assertThatThrownBy( + () -> + LakeStoragePluginSetUp.fromDataLakeFormat( + TestingClasspathLakeStoragePlugin.IDENTIFIER, + new TestingPluginManager(lakeStoragePlugins))) + .isInstanceOf(FlussRuntimeException.class) + .hasMessageContaining("Found two LakeStoragePlugin for datalake format") + .hasMessageContaining(TestingClasspathLakeStoragePlugin.IDENTIFIER) + .hasMessageContaining("/lib"); + } + + @Test + void testPluginFoundOnClasspathOnly() { + LakeStoragePlugin lakeStoragePlugin = + LakeStoragePluginSetUp.fromDataLakeFormat( + TestingClasspathLakeStoragePlugin.IDENTIFIER, null); + + assertThat(lakeStoragePlugin).isInstanceOf(PluginLakeStorageWrapper.class); + assertThat(lakeStoragePlugin.identifier()) + .isEqualTo(TestingClasspathLakeStoragePlugin.IDENTIFIER); + } + private static class TestingPluginManager implements PluginManager { private final Map, Iterator> plugins; diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/TestingClasspathLakeStoragePlugin.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/TestingClasspathLakeStoragePlugin.java new file mode 100644 index 00000000000..2e4c3f82032 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/TestingClasspathLakeStoragePlugin.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.lakestorage; + +import org.apache.fluss.config.Configuration; + +/** A {@link LakeStoragePlugin} registered via SPI on the main classpath. */ +public class TestingClasspathLakeStoragePlugin implements LakeStoragePlugin { + + public static final String IDENTIFIER = "test-classpath-plugin"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public LakeStorage createLakeStorage(Configuration configuration) { + throw new UnsupportedOperationException("Not implemented"); + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/plugin/PluginDirectoryLayoutTest.java b/fluss-common/src/test/java/org/apache/fluss/plugin/PluginDirectoryLayoutTest.java new file mode 100644 index 00000000000..6e367d5a6c7 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/plugin/PluginDirectoryLayoutTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.plugin; + +import org.apache.fluss.plugin.jar.plugina.DynamicClassA; +import org.apache.fluss.plugin.jar.plugina.TestServiceA; +import org.apache.fluss.shaded.guava32.com.google.common.collect.Lists; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.fluss.utils.Preconditions.checkState; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Tests for the plugin directory layout of a deployed cluster, where each plugin lives in its own + * sub directory of {@code /plugins} and the main classpath is {@code /lib}. + */ +class PluginDirectoryLayoutTest extends PluginTestBase { + + @TempDir private Path tmp; + + @Test + void testMultiplePluginsLoadedSimultaneously() throws Exception { + PluginManager pluginManager = + createPluginManager( + new String[] {TestSpi.class.getName(), OtherTestSpi.class.getName()}, + "multi"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + + assertThat(plugins).hasSize(2); + Set classLoaders = new HashSet<>(); + for (TestSpi plugin : plugins) { + assertThat(plugin.testMethod()).isNotNull(); + assertThat(plugin.getClassLoader()).isNotSameAs(PARENT_CLASS_LOADER); + classLoaders.add(plugin.getClassLoader()); + } + assertThat(classLoaders).hasSize(2); + } + + @Test + void testPluginsDoNotSeeEachOthersClasses() throws Exception { + PluginManager pluginManager = + createPluginManager( + new String[] {TestSpi.class.getName(), OtherTestSpi.class.getName()}, + "isolation"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + assertThat(plugins).hasSize(2); + + // DynamicClassA is bundled in plugin A's jar only + ClassLoader loaderOfA = classLoaderOf(plugins, TestServiceA.class.getName()); + ClassLoader loaderOfB = otherClassLoader(plugins, loaderOfA); + + assertThat(loaderOfA.loadClass(DynamicClassA.class.getName())).isNotNull(); + assertThatThrownBy(() -> loaderOfB.loadClass(DynamicClassA.class.getName())) + .isInstanceOf(ClassNotFoundException.class); + } + + /** The main classpath copy wins, leaving the dependencies bundled next to the plugin unused. */ + @Test + void testClassOnMainClasspathShadowsPluginCopy() throws Exception { + // TestServiceA is on the main classpath, standing in for a jar also copied into lib + PluginManager pluginManager = + createPluginManager( + new String[] { + TestSpi.class.getName(), + OtherTestSpi.class.getName(), + TestServiceA.class.getName() + }, + "shadowed"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + TestSpi fromPluginA = pluginNamed(plugins, TestServiceA.class.getName()); + + assertThat(fromPluginA).isInstanceOf(TestServiceA.class); + assertThat(fromPluginA.getClassLoader()).isSameAs(PARENT_CLASS_LOADER); + } + + @Test + void testPluginCopyIsUsedWhenNotOnMainClasspath() throws Exception { + PluginManager pluginManager = + createPluginManager( + new String[] {TestSpi.class.getName(), OtherTestSpi.class.getName()}, + "not-shadowed"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + TestSpi fromPluginA = pluginNamed(plugins, TestServiceA.class.getName()); + + // same class name, but a distinct class loaded from the plugin jar + assertThat(fromPluginA).isNotInstanceOf(TestServiceA.class); + assertThat(fromPluginA.getClassLoader()).isNotSameAs(PARENT_CLASS_LOADER); + } + + /** Builds a {@code plugins/} root holding plugin A and plugin B in separate sub directories. */ + private PluginManager createPluginManager(String[] parentPatterns, String name) + throws Exception { + File pluginRootFolder = new File(tmp.toFile(), name); + File pluginAFolder = new File(pluginRootFolder, "A"); + File pluginBFolder = new File(pluginRootFolder, "B"); + checkState(pluginAFolder.mkdirs()); + checkState(pluginBFolder.mkdirs()); + Files.copy(locateJarFile(PLUGIN_A).toPath(), Paths.get(pluginAFolder.toString(), PLUGIN_A)); + Files.copy(locateJarFile(PLUGIN_B).toPath(), Paths.get(pluginBFolder.toString(), PLUGIN_B)); + + Collection descriptors = + new DirectoryBasedPluginFinder(pluginRootFolder.toPath()).findPlugins(); + checkState(descriptors.size() == 2); + return new DefaultPluginManager(descriptors, PARENT_CLASS_LOADER, parentPatterns); + } + + private static TestSpi pluginNamed(List plugins, String className) { + for (TestSpi plugin : plugins) { + if (plugin.getClass().getName().equals(className)) { + return plugin; + } + } + throw new AssertionError("No plugin found with class name " + className); + } + + private static ClassLoader classLoaderOf(List plugins, String className) { + return pluginNamed(plugins, className).getClassLoader(); + } + + private static ClassLoader otherClassLoader(List plugins, ClassLoader classLoader) { + for (TestSpi plugin : plugins) { + if (plugin.getClassLoader() != classLoader) { + return plugin.getClassLoader(); + } + } + throw new AssertionError("Expected a second, distinct plugin class loader"); + } +} diff --git a/fluss-common/src/test/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin b/fluss-common/src/test/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin new file mode 100644 index 00000000000..d29c0f9ea0b --- /dev/null +++ b/fluss-common/src/test/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.fluss.lake.lakestorage.TestingClasspathLakeStoragePlugin diff --git a/website/docs/install-deploy/deploying-streaming-lakehouse.md b/website/docs/install-deploy/deploying-streaming-lakehouse.md index e3752fc6aa8..d5316ab69ed 100644 --- a/website/docs/install-deploy/deploying-streaming-lakehouse.md +++ b/website/docs/install-deploy/deploying-streaming-lakehouse.md @@ -124,6 +124,14 @@ See [set_cluster_configs](../engine-flink/procedures.md#set_cluster_configs) for Add JARs to `${FLUSS_HOME}/plugins//` based on your configuration: +:::warning +These JARs go in `${FLUSS_HOME}/plugins//`, together with the `fluss-lake-` JAR and +the dependencies it needs. Do not also place them in `${FLUSS_HOME}/lib`: a copy on the main +classpath shadows the plugin and is loaded without its bundled dependencies, so the CoordinatorServer +fails to start with errors such as `NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration`. +See [Troubleshooting Plugins](../maintenance/troubleshooting-plugins.md). +::: + @@ -169,7 +177,8 @@ The Tiering Service is a Flink job that continuously tiers data from Fluss to th ### Flink JARs -Add the following to `${FLINK_HOME}/lib`: +Add the following to `${FLINK_HOME}/lib` — this is the **Flink** installation, not +`${FLUSS_HOME}/lib`: diff --git a/website/docs/maintenance/tiered-storage/filesystems/overview.md b/website/docs/maintenance/tiered-storage/filesystems/overview.md index c02abbda67d..fa52dfe0ca9 100644 --- a/website/docs/maintenance/tiered-storage/filesystems/overview.md +++ b/website/docs/maintenance/tiered-storage/filesystems/overview.md @@ -30,7 +30,7 @@ Never use local file system as remote storage in production as it is not fault-t ## Pluggable File Systems The Fluss project supports the following file systems: -- **[HDFS](hdfs.md)** is supported by `fluss-fs-hadoop` and registered under the `hdfs://` URI scheme. HDFS filesystem is included in default Fluss binary distribution, so you can use it directly without manual installation. +- **[HDFS](hdfs.md)** is supported by `fluss-fs-hdfs` and registered under the `hdfs://` URI scheme. HDFS filesystem is included in default Fluss binary distribution, so you can use it directly without manual installation. `fluss-fs-hdfs` bundles its own Hadoop; the thin `fluss-fs-hadoop` artifact provides the same scheme for setups that supply Hadoop through `HADOOP_CLASSPATH`. Install exactly one of them, see [Choosing an HDFS JAR](/downloads#choosing-an-hdfs-jar). - **[Aliyun OSS](oss.md)** is supported by `fluss-fs-oss` and registered under the `oss://` URI scheme. OSS filesystem is included in default Fluss binary distribution, so you can use it directly without manual installation. @@ -42,4 +42,17 @@ The Fluss project supports the following file systems: - **[Tencent Cloud COS](cos.md)** is supported by `fluss-fs-cos` and registered under the `cosn://` URI scheme. Please make sure to [manually install the COS plugin](cos.md#install-cos-plugin-manually). -The implementation is based on [Hadoop Project](https://hadoop.apache.org/) but is self-contained with no dependency footprint. \ No newline at end of file +The implementation is based on [Hadoop Project](https://hadoop.apache.org/) but is self-contained with no dependency footprint. + +## Installing a Filesystem Plugin + +A filesystem plugin JAR goes into its own sub directory of `${FLUSS_HOME}/plugins`, named after the +URI scheme, for example `${FLUSS_HOME}/plugins/s3/`. Each sub directory is loaded in an isolated +class loader together with the dependencies next to it. + +:::warning +Never copy a plugin JAR into `${FLUSS_HOME}/lib` as well. The copy on the main classpath shadows the +one in the plugins directory and is loaded without its bundled dependencies, which fails at runtime +with errors such as `NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration`. See +[Troubleshooting Plugins](../../troubleshooting-plugins.md). +::: \ No newline at end of file diff --git a/website/docs/maintenance/troubleshooting-plugins.md b/website/docs/maintenance/troubleshooting-plugins.md new file mode 100644 index 00000000000..9f12a3eb8ea --- /dev/null +++ b/website/docs/maintenance/troubleshooting-plugins.md @@ -0,0 +1,126 @@ +--- +title: Troubleshooting Plugins +sidebar_position: 6 +--- + +# Troubleshooting Plugins + +Fluss loads filesystem, lake format and metric reporter implementations as *plugins*. Most plugin +problems come down to a single question: **is the JAR in the right directory?** This page explains +how plugins are loaded and how to recognise the errors that follow when they are not. + +## How plugins are loaded + +A Fluss installation has two very different places to put a JAR: + +``` +${FLUSS_HOME}/ +├── lib/ # the main classpath, shared by everything +└── plugins/ # one isolated sub directory per plugin + ├── hdfs/ + │ └── fluss-fs-hdfs-.jar + ├── s3/ + │ └── fluss-fs-s3-.jar + └── paimon/ + ├── fluss-lake-paimon-.jar + ├── paimon-bundle-.jar + └── flink-shaded-hadoop-2-uber-.jar +``` + +Every sub directory of `plugins/` is loaded in its own class loader, together with the dependencies +that sit next to it. That is what lets the S3 plugin and the Paimon plugin each carry their own, +possibly conflicting, versions of a library without interfering. + +The plugins directory defaults to `plugins` and can be overridden with the `FLUSS_PLUGINS_DIR` +environment variable. + +:::warning +Do not copy a plugin JAR into `${FLUSS_HOME}/lib`. Classes in `org.apache.fluss.` and +`org.apache.hadoop.` are loaded *parent-first*, so a copy on the main classpath wins over the copy +in the plugins directory — but it is loaded **without** the dependencies bundled next to the plugin. +The plugin then fails at runtime even though the JAR is clearly present. +::: + +The parent-first packages are controlled by `plugin.classloader.parent-first-patterns.default`, and +can be extended with `plugin.classloader.parent-first-patterns.additional`. + +## `${FLUSS_HOME}/lib` is not `${FLINK_HOME}/lib` + +The [Tiering Service](../streaming-lakehouse/tiering-service.md) is a **Flink** job, and its JARs — +including `fluss-lake-` — do go into `${FLINK_HOME}/lib`. That instruction is about the +Flink installation, not the Fluss one. Putting the same JAR into `${FLUSS_HOME}/lib` is what breaks +the Fluss servers. + +| JAR | Goes in | +|-----|---------| +| Filesystem plugin (`fluss-fs-*`) | `${FLUSS_HOME}/plugins//` | +| Lake format plugin (`fluss-lake-*`) on the servers | `${FLUSS_HOME}/plugins//` | +| Lake format plugin for the Tiering Service | `${FLINK_HOME}/lib` | +| Flink connector (`fluss-flink-*`) | `${FLINK_HOME}/lib` | + +## Common errors + +### Found two LakeStoragePlugin for datalake format + +``` +Found two LakeStoragePlugin for datalake format 'paimon': one in the plugins directory +and one on the main classpath (usually /lib). ... +``` + +The same lake format is installed twice. Delete the `fluss-lake-` JAR from +`${FLUSS_HOME}/lib` and keep only the one in `${FLUSS_HOME}/plugins//`, then restart the +server. This check exists so the deployment fails immediately with an explanation, rather than later +with the `NoClassDefFoundError` below. + +### NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration + +``` +java.lang.NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration + at org.apache.paimon.catalog.CatalogContext.(...) +Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hdfs.HdfsConfiguration +``` + +The Paimon plugin was loaded from the main classpath instead of from +`${FLUSS_HOME}/plugins/paimon/`, so the `flink-shaded-hadoop-2-uber` JAR bundled next to it was +never visible. Remove the copy from `${FLUSS_HOME}/lib`. + +Adding a Hadoop JAR to `${FLUSS_HOME}/lib` also makes the error disappear, but it is a workaround: +it puts Hadoop on the shared classpath for every component, which is exactly the version conflict +the plugin isolation is meant to prevent. Prefer removing the misplaced JAR. + +### Hadoop is not in the classpath, or some classes are missing + +``` +Cannot support file system for 'hdfs' via Hadoop, because Hadoop is not in the +classpath, or some classes are missing from the classpath. +``` + +The `hdfs://` scheme resolved to an implementation that has no Hadoop with it. Check that +`${FLUSS_HOME}/plugins/hdfs/` contains `fluss-fs-hdfs`, which bundles Hadoop. If you deliberately +use the thin `fluss-fs-hadoop` artifact instead, Hadoop must be supplied through `HADOOP_CLASSPATH`: + +```bash +export HADOOP_CLASSPATH=`hadoop classpath` +``` + +Do not install both artifacts — see [Choosing an HDFS JAR](/downloads#choosing-an-hdfs-jar). + +### No LakeStoragePlugin can be found for datalake format + +``` +No LakeStoragePlugin can be found for datalake format: paimon +``` + +`datalake.format` is set, but the matching plugin is not installed at all. Add the +`fluss-lake-` JAR and its dependencies to `${FLUSS_HOME}/plugins//` and restart. See +[Deploying Streaming Lakehouse](../install-deploy/deploying-streaming-lakehouse.md). + +## Checklist + +When a plugin misbehaves, check in this order: + +1. The JAR is in `${FLUSS_HOME}/plugins//`, in its own sub directory. +2. No copy of it is in `${FLUSS_HOME}/lib`. +3. The dependencies it needs sit next to it in the same sub directory. +4. Only one artifact provides each URI scheme or lake format. +5. The servers were restarted after the change — plugins are discovered at startup. diff --git a/website/src/pages/downloads.md b/website/src/pages/downloads.md index 15e84f76488..ca13e1221ed 100644 --- a/website/src/pages/downloads.md +++ b/website/src/pages/downloads.md @@ -43,4 +43,45 @@ Read the [release blog](/blog/releases/0.8/) about the new features and signific ### Verifying Downloads -Downloaded Apache Fluss (Incubating) artifacts can be verified by following [this tutorial](https://www.apache.org/info/verification.html) of the Apache Software Foundation using the Apache Fluss (Incubating) release-signing [KEYS](https://downloads.apache.org/incubator/fluss/KEYS). \ No newline at end of file +Downloaded Apache Fluss (Incubating) artifacts can be verified by following [this tutorial](https://www.apache.org/info/verification.html) of the Apache Software Foundation using the Apache Fluss (Incubating) release-signing [KEYS](https://downloads.apache.org/incubator/fluss/KEYS). + +------------------ + +## Filesystem JARs + +Fluss reaches [remote storage](/docs/maintenance/tiered-storage/remote-storage) through pluggable filesystem plugins. The binary +release already ships the HDFS, S3 and OSS plugins under `plugins//`, so those work out of +the box. The JARs below are for adding a filesystem that is not bundled, for upgrading a single +plugin in place, or for the Tiering Service running on Flink. + +:::warning +A filesystem plugin JAR belongs in `${FLUSS_HOME}/plugins//` and **must not** also be copied +into `${FLUSS_HOME}/lib`. A copy on the main classpath shadows the one in the plugins directory and +is loaded without the dependencies bundled next to it, which fails at runtime with errors such as +`NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration`. See +[Troubleshooting Plugins](/docs/next/maintenance/troubleshooting-plugins). +::: + +The following are the plugins of the latest stable release, Apache Fluss (Incubating) 0.9.1: + +| Filesystem | URI scheme | In the binary release | JAR | +|------------|------------|-----------------------|-----| +| HDFS | `hdfs://` | yes | [fluss-fs-hdfs-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-hdfs/0.9.1-incubating/fluss-fs-hdfs-0.9.1-incubating.jar) | +| AWS S3 | `s3://` | yes | [fluss-fs-s3-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-s3/0.9.1-incubating/fluss-fs-s3-0.9.1-incubating.jar) | +| Aliyun OSS | `oss://` | yes | [fluss-fs-oss-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-oss/0.9.1-incubating/fluss-fs-oss-0.9.1-incubating.jar) | +| Google Cloud Storage | `gs://` | no | [fluss-fs-gs-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-gs/0.9.1-incubating/fluss-fs-gs-0.9.1-incubating.jar) | +| Azure Blob Storage | `abfs://` | no | [fluss-fs-azure-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-azure/0.9.1-incubating/fluss-fs-azure-0.9.1-incubating.jar) | +| HuaweiCloud OBS | `obs://` | no | [fluss-fs-obs-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-obs/0.9.1-incubating/fluss-fs-obs-0.9.1-incubating.jar) | + +### Choosing an HDFS JAR + +Two artifacts provide the `hdfs://` scheme and they differ only in whether Hadoop travels with them. +Pick one, never both: + +| Artifact | Hadoop dependencies | Use it when | +|----------|---------------------|-------------| +| `fluss-fs-hdfs` | bundled, self-contained (~34 MB) | Default, and what the binary release ships. Works on machines with no Hadoop installation. | +| `fluss-fs-hadoop` | not bundled (~10 KB) | You provide Hadoop yourself through `HADOOP_CLASSPATH`, for example to match your cluster's Hadoop version or to use Kerberos. | + +Deploying both at once gives two implementations of the same scheme, so use exactly one. See +[HDFS](/docs/next/maintenance/tiered-storage/filesystems/hdfs) for the Hadoop configuration options.