diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml index 306455931901b..520fca4513af1 100644 --- a/.github/workflows/cluster-it-1c1d.yml +++ b/.github/workflows/cluster-it-1c1d.yml @@ -19,6 +19,8 @@ on: - "rel/*" - "rc/*" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/cluster-it-1c1d1a.yml b/.github/workflows/cluster-it-1c1d1a.yml index 74ebcbebaac6d..cc8e17fc82384 100644 --- a/.github/workflows/cluster-it-1c1d1a.yml +++ b/.github/workflows/cluster-it-1c1d1a.yml @@ -20,6 +20,8 @@ on: - 'rc/*' - 'force_ci/**' paths-ignore: + - '.github/workflows/**' + - 'integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java' - 'docs/**' - 'site/**' - 'iotdb-client/client-cpp/**' diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml index 14d808462f5f5..cf399f178987c 100644 --- a/.github/workflows/cluster-it-1c3d.yml +++ b/.github/workflows/cluster-it-1c3d.yml @@ -20,6 +20,8 @@ on: - "rc/*" - "force_ci/**" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index c0dcb39df8886..aae08fa57f6fb 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -22,6 +22,8 @@ on: - "rc/*" - "force_ci/**" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/edge-it.yml b/.github/workflows/edge-it.yml index e4adfdd9f0cd9..a29dbb63c46b5 100644 --- a/.github/workflows/edge-it.yml +++ b/.github/workflows/edge-it.yml @@ -19,6 +19,8 @@ on: - "rel/*" - "rc/*" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 7b5d6df0b9e11..40890bedc34ee 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -21,6 +21,8 @@ on: - "rc/*" - "force_ci/**" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/sonar-codecov.yml b/.github/workflows/sonar-codecov.yml index 88f69e5938d4f..cdb381716af8b 100644 --- a/.github/workflows/sonar-codecov.yml +++ b/.github/workflows/sonar-codecov.yml @@ -24,6 +24,8 @@ on: - "rc/*" - "force_ci/**" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml index 8c5a838af3395..469f0f9d69466 100644 --- a/.github/workflows/table-cluster-it-1c1d.yml +++ b/.github/workflows/table-cluster-it-1c1d.yml @@ -39,7 +39,7 @@ env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: - # Ubuntu runs all ITs in a single job (already fast at ~39 min) + # This draft stress-test workflow intentionally runs only IoTDBConcurrentCreateTableIT. Ubuntu: runs-on: ubuntu-latest @@ -67,7 +67,9 @@ jobs: mvn clean verify \ -P with-integration-tests \ -DskipUTs \ - -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \ + -Dit.test=IoTDBConcurrentCreateTableIT \ + -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ -pl integration-test \ -am -PTableSimpleIT - name: Upload Artifact @@ -78,12 +80,7 @@ jobs: path: integration-test/target/cluster-logs retention-days: 1 - # Windows is ~67% slower than Ubuntu, so split into 3 shards to parallelize Windows: - strategy: - fail-fast: false - matrix: - shard: [0, 1, 2] runs-on: windows-latest steps: @@ -108,41 +105,21 @@ jobs: netsh int ipv4 set dynamicport udp start=32768 num=32768 netsh int ipv6 set dynamicport tcp start=32768 num=32768 netsh int ipv6 set dynamicport udp start=32768 num=32768 - - name: Build IT shard list - shell: bash - # Distribute TableLocalStandaloneIT test classes across 3 shards using hash-mod assignment. - # The list is written to a file so failsafe.includesFile can read it without command-line length limits. - run: | - set -euo pipefail - SHARD=${{ matrix.shard }} - TOTAL=3 - # Write outside the repo so Apache RAT (license check) doesn't flag the file. - # Using a single grep -rl call instead of `find | xargs grep`: on Windows Git Bash, - # ARG_MAX is small so xargs batches the file list, and any batch with no matches - # makes grep exit 1, which makes xargs exit 123 and trips `set -o pipefail`. - grep -rl --include='*IT.java' 'TableLocalStandaloneIT' integration-test/src/test/java \ - | awk -F'/' '{print $NF}' | sed 's/\.java$//' \ - | sort \ - | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \ - > "$RUNNER_TEMP/it-shard.txt" - echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes" - head -5 "$RUNNER_TEMP/it-shard.txt" - name: IT/UT Test shell: bash run: | mvn clean verify \ -P with-integration-tests \ -DskipUTs \ - -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \ - -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \ - -DfailIfNoTests=false \ - -Dfailsafe.failIfNoSpecifiedTests=false \ + -Dit.test=IoTDBConcurrentCreateTableIT \ + -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ -pl integration-test \ -am -PTableSimpleIT - name: Upload Artifact if: failure() uses: actions/upload-artifact@v6 with: - name: table-standalone-log-Windows-shard${{ matrix.shard }} + name: table-standalone-log-Windows path: integration-test/target/cluster-logs retention-days: 1 diff --git a/.github/workflows/table-cluster-it-1c3d.yml b/.github/workflows/table-cluster-it-1c3d.yml index abca80623699c..8016a1f30db49 100644 --- a/.github/workflows/table-cluster-it-1c3d.yml +++ b/.github/workflows/table-cluster-it-1c3d.yml @@ -20,6 +20,8 @@ on: - "rc/*" - "force_ci/**" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/todos-check.yml b/.github/workflows/todos-check.yml index fc33b12d3cb58..bc768ff3179c8 100644 --- a/.github/workflows/todos-check.yml +++ b/.github/workflows/todos-check.yml @@ -9,6 +9,8 @@ on: - "rc/*" - 'force_ci/**' paths-ignore: + - '.github/workflows/**' + - 'integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java' - 'docs/**' - 'site/**' # allow manually run the action: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 64a20a1f9a25a..26ab99d896533 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -23,6 +23,8 @@ on: - "rc/*" - "force_ci/**" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/.github/workflows/zh-locale-compile.yml b/.github/workflows/zh-locale-compile.yml index 93e264cee2632..a07f652dfac91 100644 --- a/.github/workflows/zh-locale-compile.yml +++ b/.github/workflows/zh-locale-compile.yml @@ -32,6 +32,8 @@ on: - "rc/*" - "force_ci/**" paths-ignore: + - ".github/workflows/**" + - "integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java" - "docs/**" - "site/**" - "iotdb-client/client-cpp/**" diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java new file mode 100644 index 0000000000000..ba4a1a2ed23e8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBConcurrentCreateTableIT.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(IoTDBTestRunner.class) +@Category(TableLocalStandaloneIT.class) +public class IoTDBConcurrentCreateTableIT { + + private static final String DATABASE_NAME = "concurrent_create_table_db"; + private static final int THREAD_COUNT = 16; + private static final int TABLE_COUNT = 80_000; + private static final int TABLES_PER_THREAD = TABLE_COUNT / THREAD_COUNT; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testConcurrentCreateEightyThousandTables() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + } + + ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT); + CountDownLatch ready = new CountDownLatch(THREAD_COUNT); + CountDownLatch start = new CountDownLatch(1); + List> createTableTasks = new ArrayList<>(THREAD_COUNT); + + try { + for (int threadIndex = 0; threadIndex < THREAD_COUNT; threadIndex++) { + final int firstTableIndex = threadIndex * TABLES_PER_THREAD; + createTableTasks.add( + executor.submit( + () -> { + boolean readySignaled = false; + try (Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + ready.countDown(); + readySignaled = true; + assertTrue( + "Timed out waiting for all create-table threads to start", + start.await(1, TimeUnit.MINUTES)); + for (int offset = 0; offset < TABLES_PER_THREAD; offset++) { + statement.execute( + "CREATE TABLE " + + DATABASE_NAME + + ".table_" + + (firstTableIndex + offset) + + " (tag STRING TAG, value INT32 FIELD)"); + } + } finally { + if (!readySignaled) { + ready.countDown(); + } + } + return null; + })); + } + + assertTrue("Timed out preparing create-table threads", ready.await(1, TimeUnit.MINUTES)); + start.countDown(); + executor.shutdown(); + for (Future createTableTask : createTableTasks) { + createTableTask.get(2, TimeUnit.HOURS); + } + } finally { + start.countDown(); + executor.shutdownNow(); + executor.awaitTermination(30, TimeUnit.SECONDS); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery( + "SELECT count(*) FROM information_schema.tables WHERE database = '" + + DATABASE_NAME + + "'")) { + assertTrue(resultSet.next()); + assertEquals(TABLE_COUNT, resultSet.getLong(1)); + assertFalse(resultSet.next()); + } + } +}