Skip to content

Commit 4159b3a

Browse files
halibobo1205halibobo1205
committed
feat(toolkit): add db conversion and data splitting extensions
 1. convert leveldb to rocksdb 2. split lite full node data 3. optimize gradle build
1 parent 8f78620 commit 4159b3a

31 files changed

Lines changed: 2813 additions & 177 deletions

‎framework/build.gradle‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -267,14 +267,34 @@ createScript(project, 'org.tron.program.DBConvert', 'DBConvert')
267267
createScript(project, 'org.tron.tool.litefullnode.LiteFullNodeTool', 'LiteFullNodeTool')
268268

269269
def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true'
270+
def skipSolidity = hasProperty('skipSolidity') ? true : false
271+
def skipKeystore = hasProperty('skipKeystore') ? true : false
272+
def skipConvert = hasProperty('skipConvert') ? true : false
273+
def skipLite = hasProperty('skipLite') ? true : false
274+
def skipAll = hasProperty('skipAll') ? true : false
270275
if (releaseBinary == 'true') {
271276
artifacts {
272-
archives(binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode'),
273-
binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode'),
274-
binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory'),
275-
binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert'),
276-
binaryRelease('buildLiteFullNodeToolJar', 'LiteFullNodeTool', 'org.tron.tool.litefullnode.LiteFullNodeTool'))
277+
archives(binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode'))
277278
}
279+
if (!skipAll) {
280+
if (!skipSolidity) {
281+
artifacts {
282+
archives(binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode'))}
283+
}
284+
if (!skipKeystore) {
285+
artifacts {
286+
archives(binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory'))}
287+
}
288+
if (!skipConvert) {
289+
artifacts {
290+
archives(binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert'))}
291+
}
292+
if (!skipLite) {
293+
artifacts {
294+
archives(binaryRelease('buildLiteFullNodeToolJar', 'LiteFullNodeTool', 'org.tron.tool.litefullnode.LiteFullNodeTool'))}
295+
}
296+
}
297+
278298
}
279299

280300
task copyToParent(type: Copy) {

‎framework/src/main/java/org/tron/core/config/args/Args.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1437,7 +1437,7 @@ public static boolean checkIsLiteFullNode() {
14371437
PARAMETER.storage.getDbDirectory(), Constant.INFO_FILE_NAME).toString();
14381438
if (FileUtil.isExists(infoFile)) {
14391439
String value = PropUtil.readProperty(infoFile, Constant.SPLIT_BLOCK_NUM);
1440-
return !"".equals(value) && Long.parseLong(value) > 0;
1440+
return !"".equals(value) && Long.parseLong(value) > 1;
14411441
}
14421442
return false;
14431443
}

‎framework/src/main/java/org/tron/tool/litefullnode/DbTool.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
public class DbTool {
2828

2929
private static final String KEY_ENGINE = "ENGINE";
30-
private static final String ENGINE_FILE = "engine.properties";
30+
public static final String ENGINE_FILE = "engine.properties";
3131
private static final String FILE_SEPARATOR = File.separator;
3232
private static final String ROCKSDB = "ROCKSDB";
3333

‎framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.io.FileNotFoundException;
1515
import java.io.IOException;
1616
import java.nio.file.Files;
17+
import java.nio.file.Path;
1718
import java.nio.file.Paths;
1819
import java.util.Arrays;
1920
import java.util.List;
@@ -331,6 +332,17 @@ private void fillSnapshotBlockAndTransDb(String sourceDir, String snapshotDir)
331332
DBInterface destCommonDb = DbTool.getDB(snapshotDir, COMMON_DB_NAME);
332333
destCommonDb.put(DB_KEY_NODE_TYPE, ByteArray.fromInt(Constant.NODE_TYPE_LIGHT_NODE));
333334
destCommonDb.put(DB_KEY_LOWEST_BLOCK_NUM, ByteArray.fromLong(startIndex));
335+
// copy engine.properties for block、block-index、trans from source if exist
336+
copyEngineIfExist(sourceDir, snapshotDir, BLOCK_DB_NAME, BLOCK_INDEX_DB_NAME, TRANS_DB_NAME);
337+
}
338+
339+
private void copyEngineIfExist(String source, String dest, String... dbNames) {
340+
for (String dbName : dbNames) {
341+
Path ori = Paths.get(source, dbName, DbTool.ENGINE_FILE);
342+
if (ori.toFile().exists()) {
343+
Util.copy(ori, Paths.get(dest, dbName, DbTool.ENGINE_FILE));
344+
}
345+
}
334346
}
335347

336348
private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDBException {

‎framework/src/main/java/org/tron/tool/litefullnode/Util.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public static void copyDatabases(Path src, Path dest, List<String> subDirs)
4343
});
4444
}
4545

46-
private static void copy(Path source, Path dest) {
46+
public static void copy(Path source, Path dest) {
4747
try {
4848
// create hard link when file is .sst
4949
if (source.toString().endsWith(".sst")) {

‎plugins/README.md‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Toolkit Manual
2+
3+
This package contains a set of tools for Tron, the following is the documentation for each tool.
4+
5+
## DB Archive
6+
7+
DB archive provides the ability to reformat the manifest according to the current `database`,
8+
parameters are compatible with previous `ArchiveManifest`.
9+
10+
Parameter explanation:
11+
12+
- `-b | --batch-size`: [int] specify the batch manifest size,default：80000.
13+
- `-d | --database-directory`: [string] specify the database directory to be processed,default：output-directory/database.
14+
- `-m | --manifest-size`: [int] specify the minimum required manifest file size ，unit:M，default：0.
15+
- `-h | --help`: provide the help info
16+
17+
Demo:
18+
19+
```shell script
20+
# full command
21+
java -jar Toolkit.jar db archive [-h] [-b=<maxBatchSize>] [-d=<databaseDirectory>] [-m=<maxManifestSize>]
22+
# examples
23+
java -jar Toolkit.jar db archive #1. use default settings
24+
java -jar Toolkit.jar db archive -d /tmp/db/database #2. Specify the database directory as /tmp/db/database
25+
java -jar Toolkit.jar db archive -b 64000 #3. Specify the batch size to 64000 when optimizing Manifest
26+
java -jar Toolkit.jar db archive -m 128 #4. Specify optimization only when Manifest exceeds 128M
27+
```
28+
29+
30+
## DB Convert
31+
32+
DB convert provides a helper which can convert LevelDB data to RocksDB data, parameters are compatible with previous `DBConvert`.
33+
34+
Parameter explanation:
35+
36+
- `<src>`: Input path for leveldb. Default: output-directory/database
37+
- `<dest>`: Output path for rocksdb. Default: output-directory-dst/database
38+
- `--safe`: In safe mode, read data from leveldb then put rocksdb, it's a very time-consuming procedure. If not, just change engine.properties from leveldb to rocksdb,rocksdb
39+
is compatible with leveldb for current version.This may not be the case in the future.Default: false
40+
- `-h | --help`: provide the help info
41+
42+
Demo:
43+
44+
```shell script
45+
# full command
46+
java -jar Toolkit.jar db convert [-h] [--safe] <src> <dest>
47+
# examples
48+
java -jar Toolkit.jar db convert output-directory/database /tmp/databse
49+
```
50+
51+
52+
## DB Lite
53+
54+
DB Lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`.
55+
56+
Parameter explanation:
57+
58+
- `-o | --operate`: [split,merge]. Default: split.
59+
- `-t | --type`: only used with operate=split: [snapshot,history]. Default: snapshot.
60+
- `-fn | --fn-data-path`: the database path to be split or merged.
61+
- `-ds | --dataset-path`: when operation is `split`,`dataset-path` is the path that store the `snapshot` or `history`,when
62+
operation is `split`,`dataset-path` is the `history` data path.
63+
- `-h | --help`: provide the help info
64+
65+
Demo:
66+
67+
```shell script
68+
# full command
69+
java -jar Toolkit.jar db lite [-h] -ds=<datasetPath> -fn=<fnDataPath> [-o=<operate>] [-t=<type>]
70+
# examples
71+
#split and get a snapshot dataset
72+
java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp
73+
#split and get a history dataset
74+
java -jar Toolkit.jar db lite -o split -t history --fn-data-path output-directory/database --dataset-path /tmp
75+
#merge history dataset and snapshot dataset
76+
java -jar Toolkit.jar db lite -o split -t history --fn-data-path /tmp/snapshot --dataset-path /tmp/history
77+
```
78+
79+
## DB Move
80+
81+
DB Move provides a helper to move some dbs to pre-set new path. For example move `block`, `transactionRetStore` or `transactionHistoryStore` to HDD,reduce storage expenses
82+
83+
Parameter explanation:
84+
85+
- `-c | --config`: config file. Default: config.conf.
86+
- `-d | --database-directory`: database directory path. Default: output-directory.
87+
- `-h | --help`: provide the help info
88+
89+
Demo:
90+
91+
Take the example of moving `block` and `trans`
92+
93+
```conf
94+
storage {
95+
......
96+
properties = [
97+
{
98+
name = "block",
99+
path = "/data1/tron",
100+
},
101+
{
102+
name = "trans",
103+
path = "/data1/tron",
104+
}
105+
]
106+
......
107+
}
108+
```
109+
110+
```shell script
111+
# full command
112+
java -jar Toolkit.jar db mv [-h] [-c=<config>] [-d=<database>]
113+
# examples
114+
java -jar Toolkit.jar db mv -c main_net_config.conf -d /data/tron/output-directory
115+
```

‎plugins/build.gradle‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,11 @@ dependencies {
3333
compile group: 'info.picocli', name: 'picocli', version: '4.6.3'
3434
compile group: 'com.typesafe', name: 'config', version: '1.3.2'
3535
compile group: 'me.tongfei', name: 'progressbar', version: '0.9.3'
36-
37-
38-
compile 'com.github.halibobo1205.leveldb-java:leveldb:v0.12.5'
39-
compile 'com.github.halibobo1205.leveldb-java:leveldb-api:v0.12.5'
36+
compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69'
37+
compile group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10'
38+
compile 'com.halibobor:leveldbjni-all:1.18.2'
39+
compile 'com.halibobor:leveldb:1.18.2'
40+
compile project(":protocol")
4041
}
4142

4243
check.dependsOn 'lint'
@@ -104,6 +105,11 @@ def binaryRelease(taskName, jarName, mainClass) {
104105
}
105106
}
106107

108+
// exclude these files for bouncycastle
109+
exclude "META-INF/*.SF"
110+
exclude "META-INF/*.DSA"
111+
exclude "META-INF/*.RSA"
112+
107113
manifest {
108114
attributes "Main-Class": "${mainClass}"
109115
}
@@ -132,11 +138,18 @@ createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest')
132138
createScript(project, 'org.tron.plugins.Toolkit', 'Toolkit')
133139

134140
def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true'
141+
def skipArchive = hasProperty('skipArchive') ? true : false
142+
def skipAll = hasProperty('skipAll') ? true : false
135143
if (releaseBinary == 'true') {
136144
artifacts {
137-
archives(
138-
binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest'),
139-
binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit'))
145+
archives(binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit'))
146+
}
147+
if (!skipAll) {
148+
if (!skipArchive) {
149+
artifacts {
150+
archives(binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest'))
151+
}
152+
}
140153
}
141154
}
142155

‎plugins/src/main/java/org/tron/plugins/ArchiveManifest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,6 @@ public static org.iq80.leveldb.Options newDefaultLevelDbOptions() {
8282
dbOptions.maxOpenFiles(1000);
8383
dbOptions.maxBatchSize(64_000);
8484
dbOptions.maxManifestSize(128);
85-
dbOptions.fast(false);
8685
return dbOptions;
8786
}
8887

‎plugins/src/main/java/org/tron/plugins/Db.java‎

Lines changed: 7 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,18 @@
11
package org.tron.plugins;
22

3-
import com.typesafe.config.Config;
4-
import com.typesafe.config.ConfigFactory;
5-
import java.io.File;
6-
import java.io.IOException;
7-
import java.nio.file.Path;
8-
import java.nio.file.Paths;
93
import picocli.CommandLine;
104

115
@CommandLine.Command(name = "db",
126
mixinStandardHelpOptions = true,
137
version = "db command 1.0",
148
description = "An rich command set that provides high-level operations for dbs.",
15-
subcommands = {CommandLine.HelpCommand.class, DbMove.class, DbArchive.class},
16-
commandListHeading = "%nCommands:%n%nThe most commonly used git commands are:%n"
9+
subcommands = {CommandLine.HelpCommand.class,
10+
DbMove.class,
11+
DbArchive.class,
12+
DbConvert.class,
13+
DbLite.class
14+
},
15+
commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n"
1716
)
1817
public class Db {
19-
20-
static class ConfigConverter implements CommandLine.ITypeConverter<Config> {
21-
ConfigConverter() {
22-
}
23-
24-
public Config convert(String value) throws IOException {
25-
File file = Paths.get(value).toFile();
26-
if (file.exists() && file.isFile()) {
27-
return ConfigFactory.parseFile(Paths.get(value).toFile());
28-
} else {
29-
throw new IOException("DB config [" + value + "] not exist!");
30-
}
31-
}
32-
}
33-
34-
static class PathConverter implements CommandLine.ITypeConverter<Path> {
35-
PathConverter() {
36-
}
37-
38-
public Path convert(String value) throws IOException {
39-
File file = Paths.get(value).toFile();
40-
if (file.exists() && file.isDirectory()) {
41-
return file.toPath();
42-
} else {
43-
throw new IOException("DB path [" + value + "] not exist!");
44-
}
45-
}
46-
}
4718
}

0 commit comments

Comments
 (0)