diff --git a/plugins/src/main/java/common/org/tron/plugins/DbLite.java b/plugins/src/main/java/common/org/tron/plugins/DbLite.java index 3f8a6cb58c8..18bbcdb6958 100644 --- a/plugins/src/main/java/common/org/tron/plugins/DbLite.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbLite.java @@ -6,6 +6,7 @@ import com.google.common.primitives.Bytes; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; +import com.google.protobuf.ByteString; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -30,6 +31,7 @@ import org.tron.plugins.utils.db.DBIterator; import org.tron.plugins.utils.db.DbTool; import org.tron.protos.Protocol; +import org.tron.protos.contract.BalanceContract; import picocli.CommandLine; @Slf4j(topic = "lite") @@ -57,13 +59,17 @@ public class DbLite implements Callable { private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore"; private static final String PROPERTIES_DB_NAME = "properties"; private static final String TRANS_CACHE_DB_NAME = "trans-cache"; + private static final String BALANCE_TRACE_DB_NAME = "balance-trace"; + private static final String ACCOUNT_TRACE_DB_NAME = "account-trace"; private static final List archiveDbs = Arrays.asList( BLOCK_DB_NAME, BLOCK_INDEX_DB_NAME, TRANS_DB_NAME, TRANSACTION_RET_DB_NAME, - TRANSACTION_HISTORY_DB_NAME); + TRANSACTION_HISTORY_DB_NAME, + BALANCE_TRACE_DB_NAME, + ACCOUNT_TRACE_DB_NAME); enum Operate { split, merge } @@ -522,24 +528,42 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo) DBInterface blockDb = DbTool.getDB(liteDir, BLOCK_DB_NAME); DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME); DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME); - + DBInterface balanceTraceDb = DbTool.getDB(liteDir, BALANCE_TRACE_DB_NAME); + DBInterface accountTraceDb = DbTool.getDB(liteDir, ACCOUNT_TRACE_DB_NAME); ProgressBar.wrap(LongStream.rangeClosed(start, end) .boxed() - .sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> { + .sorted((a, b) -> Long.compare(b, a)), "trimHistory") + .map(ByteArray::fromLong).forEach(n -> { try { - byte[] blockIdHash = blockIndexDb.get(ByteArray.fromLong(n)); + byte[] blockIdHash = blockIndexDb.get(n); Protocol.Block block = Protocol.Block.parseFrom(blockDb.get(blockIdHash)); // delete transactions for (Protocol.Transaction e : block.getTransactionsList()) { transDb.delete(DBUtils.getTransactionId(e).getBytes()); } // delete transaction result - tranRetDb.delete(ByteArray.fromLong(n)); + tranRetDb.delete(n); // delete block blockDb.delete(blockIdHash); // delete block index - blockIndexDb.delete(ByteArray.fromLong(n)); + blockIndexDb.delete(n); + byte[] balanceTrace = balanceTraceDb.get(n); + if (balanceTrace != null) { + long blockNum = ByteArray.toLong(n); + // delete account trace: one row per (address, blockNum) touched in this block + BalanceContract.BlockBalanceTrace.parseFrom(balanceTrace) + .getTransactionBalanceTraceList().stream() + .flatMap(tx -> tx.getOperationList().stream()) + .map(BalanceContract.TransactionBalanceTrace.Operation::getAddress) + .distinct() + .map(ByteString::toByteArray) + .map(address -> Bytes.concat(address, Longs.toByteArray( + blockNum ^ Long.MAX_VALUE))) + .forEach(accountTraceDb::delete); + // delete balance trace + balanceTraceDb.delete(n); + } } catch (IOException e) { throw new RuntimeException(e); } @@ -570,7 +594,13 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws DBInterface bakDb = DbTool.getDB(bakDir.toString(), dbName); DBInterface destDb = DbTool.getDB(liteDir, dbName); try (DBIterator iterator = bakDb.iterator()) { - if (TRANS_DB_NAME.equals(dbName) || TRANSACTION_HISTORY_DB_NAME.equals(dbName)) { + // These DBs cannot be positioned by block number: + // TRANS_DB_NAME / TRANSACTION_HISTORY_DB_NAME are keyed by txId; + // ACCOUNT_TRACE_DB_NAME is keyed by address || (blockNum ^ Long.MAX_VALUE). + // Full replay is safe: the overlapping segment [0, historyMaxNum] is + // identical on both sides, so re-putting it is idempotent. + if (TRANS_DB_NAME.equals(dbName) || TRANSACTION_HISTORY_DB_NAME.equals(dbName) + || ACCOUNT_TRACE_DB_NAME.equals(dbName)) { iterator.seekToFirst(); } else { iterator.seek(head); diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java index 60db838a80d..7405ba5f362 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -1,16 +1,20 @@ package org.tron.plugins; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.tron.common.utils.PublicMethod.getRandomPrivateKey; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; import org.tron.api.WalletGrpc; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; @@ -23,6 +27,9 @@ import org.tron.common.utils.Utils; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j @@ -89,7 +96,13 @@ public void clear() { public void testTools(String dbType, int checkpointVersion) throws InterruptedException, IOException { - logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); + testTools(dbType, checkpointVersion, false, false); + } + + public void testTools(String dbType, int checkpointVersion, boolean advanceSnapshot, + boolean historyBalanceLookup) throws InterruptedException, IOException { + logger.info("dbType {}, checkpointVersion {}, advanceSnapshot {}, historyBalanceLookup {}", + dbType, checkpointVersion, advanceSnapshot, historyBalanceLookup); dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); init(dbType); final String[] argsForSnapshot = @@ -104,6 +117,7 @@ public void testTools(String dbType, int checkpointVersion) new String[] {"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath + File.separator + "history"}; Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); + Args.getInstance().setHistoryBalanceLookup(historyBalanceLookup); DbLite.setRecentBlks(3); // start fullNode startApp(); @@ -115,14 +129,26 @@ public void testTools(String dbType, int checkpointVersion) FileUtil.deleteDir(Paths.get(dbPath, databaseDir, "trans-cache").toFile()); // generate snapshot cli.execute(argsForSnapshot); + // snapshot must not ship balance-trace / account-trace (packaging contract for #6597) + Path snapshotDir = Paths.get(dbPath, "snapshot"); + assertFalse(snapshotDir.resolve("balance-trace").toFile().exists()); + assertFalse(snapshotDir.resolve("account-trace").toFile().exists()); // start fullNode startApp(); - // produce transactions - generateSomeTransactions(checkpointVersion == 1 ? 6 : 18); + int postSnapshotTxSeconds = advanceSnapshot && checkpointVersion == 1 ? 6 : 18; + generateSomeTransactions(postSnapshotTxSeconds); // stop the node shutdown(); // generate history cli.execute(argsForHistory); + // history carries balance-trace / account-trace regardless of lookup setting + Path historyDir = Paths.get(dbPath, "history"); + assertTrue(historyDir.resolve("balance-trace").toFile().exists()); + assertTrue(historyDir.resolve("account-trace").toFile().exists()); + if (historyBalanceLookup) { + assertDbHasEntries(historyDir, "balance-trace"); + assertDbHasEntries(historyDir, "account-trace"); + } // backup original database to database_bak File database = new File(Paths.get(dbPath, databaseDir).toString()); if (!database.renameTo(new File(Paths.get(dbPath, databaseDir + "_bak").toString()))) { @@ -137,11 +163,11 @@ public void testTools(String dbType, int checkpointVersion) String.format("rename snapshot to %s failed", Paths.get(dbPath, databaseDir))); } - // start and validate the snapshot - startApp(); - generateSomeTransactions(checkpointVersion == 1 ? 18 : 6); - // stop the node - shutdown(); + if (advanceSnapshot) { + startApp(); + generateSomeTransactions(checkpointVersion == 1 ? 18 : 6); + shutdown(); + } // merge history cli.execute(argsForMerge); // start and validate @@ -151,6 +177,18 @@ public void testTools(String dbType, int checkpointVersion) DbLite.reSetRecentBlks(); } + private static void assertDbHasEntries(Path parent, String dbName) { + try { + DBInterface db = DbTool.getDB(parent.toString(), dbName); + try (DBIterator it = db.iterator()) { + it.seekToFirst(); + assertTrue(dbName + " should have at least one entry", it.hasNext()); + } + } catch (IOException | RocksDBException e) { + throw new RuntimeException(e); + } + } + private void generateSomeTransactions(int during) { during *= 1000; // ms int runTime = 0; diff --git a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java index ab1067fefc3..77d74e26096 100644 --- a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java @@ -8,6 +8,6 @@ public class DbLiteRocksDbV2Test extends DbLiteTest { @Test public void testToolsWithRocksDB() throws InterruptedException, IOException { - testTools("ROCKSDB", 2); + testTools("ROCKSDB", 2, true, false); } } diff --git a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbTest.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbTest.java new file mode 100644 index 00000000000..c182b0a5d84 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbTest.java @@ -0,0 +1,13 @@ +package org.tron.plugins.rocksdb; + +import java.io.IOException; +import org.junit.Test; +import org.tron.plugins.DbLiteTest; + +public class DbLiteWithHistoryRocksDbTest extends DbLiteTest { + + @Test + public void testToolsWithTrimHistoryV1() throws InterruptedException, IOException { + testTools("ROCKSDB", 1, true, true); + } +} diff --git a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbV2Test.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbV2Test.java new file mode 100644 index 00000000000..b4ba9ebf04f --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbV2Test.java @@ -0,0 +1,13 @@ +package org.tron.plugins.rocksdb; + +import java.io.IOException; +import org.junit.Test; +import org.tron.plugins.DbLiteTest; + +public class DbLiteWithHistoryRocksDbV2Test extends DbLiteTest { + + @Test + public void testToolsWithTrimHistoryV2() throws InterruptedException, IOException { + testTools("ROCKSDB", 2, true, true); + } +}