Skip to content

Commit 68432ea

Browse files
halibobo1205halibobo1205
committed
plugins for archive manifest
1 parent bed59a6 commit 68432ea

5 files changed

Lines changed: 572 additions & 0 deletions

File tree

‎plugins/build.gradle‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
plugins {
2+
id "org.sonarqube" version "2.6"
3+
}
4+
5+
apply plugin: 'application'
6+
apply plugin: 'checkstyle'
7+
8+
jacoco {
9+
toolVersion = "0.8.4"
10+
}
11+
def versions = [
12+
checkstyle: '8.7',
13+
]
14+
mainClassName = 'org.tron.plugins.ArchiveManifest'
15+
group 'org.tron'
16+
version '1.0.0'
17+
18+
configurations {
19+
checkstyleConfig
20+
}
21+
22+
configurations.getByName('checkstyleConfig') {
23+
transitive = false
24+
}
25+
26+
task version(type: Exec) {
27+
commandLine 'bash', '-c', '../ver.sh'
28+
}
29+
dependencies {
30+
//local libraries
31+
compile fileTree(dir: 'libs', include: '*.jar')
32+
testCompile group: 'junit', name: 'junit', version: '4.12'
33+
testCompile group: 'org.mockito', name: 'mockito-core', version: '2.13.0'
34+
testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1'
35+
testCompile group: 'org.testng', name: 'testng', version: '6.14.3'
36+
// https://mvnrepository.com/artifact/com.beust/jcommander
37+
compile group: 'com.beust', name: 'jcommander', version: '1.78'
38+
39+
compile 'com.github.halibobo1205.leveldb-java:leveldb:v0.12.5'
40+
compile 'com.github.halibobo1205.leveldb-java:leveldb-api:v0.12.5'
41+
}
42+
43+
check.dependsOn 'lint'
44+
45+
checkstyle {
46+
toolVersion = "${versions.checkstyle}"
47+
configFile = file("../framework/config/checkstyle/checkStyleAll.xml")
48+
}
49+
50+
checkstyleMain {
51+
source = 'src/main/java'
52+
}
53+
54+
task lint(type: Checkstyle) {
55+
// Cleaning the old log because of the creation of the new ones (not sure if totaly needed)
56+
delete fileTree(dir: "${project.rootDir}/app/build/reports")
57+
source 'src'
58+
include '**/*.java'
59+
exclude 'main/gen/**'
60+
exclude 'test/**'
61+
// empty classpath
62+
classpath = files()
63+
//Failing the build
64+
ignoreFailures = false
65+
}
66+
67+
tasks.matching { it instanceof Test }.all {
68+
testLogging.events = ["failed", "passed", "skipped"]
69+
}
70+
71+
if (project.hasProperty("mainClass")) {
72+
mainClassName = mainClass
73+
}
74+
75+
test {
76+
testLogging {
77+
exceptionFormat = 'full'
78+
}
79+
jacoco {
80+
destinationFile = file("$buildDir/jacoco/jacocoTest.exec")
81+
classDumpDir = file("$buildDir/jacoco/classpathdumps")
82+
}
83+
}
84+
85+
jacocoTestReport {
86+
reports {
87+
xml.enabled true
88+
csv.enabled false
89+
html.destination file("${buildDir}/jacocoHtml")
90+
}
91+
executionData.from = 'build/jacoco/jacocoTest.exec'
92+
}
93+
94+
def binaryRelease(taskName, jarName, mainClass) {
95+
return tasks.create("${taskName}", Jar) {
96+
baseName = jarName
97+
version = null
98+
from(sourceSets.main.output) {
99+
include "/**"
100+
}
101+
102+
from {
103+
configurations.compile.collect {
104+
it.isDirectory() ? it : zipTree(it)
105+
}
106+
}
107+
108+
manifest {
109+
attributes "Main-Class": "${mainClass}"
110+
}
111+
}
112+
}
113+
114+
def createScript(project, mainClass, name) {
115+
project.tasks.create(name: name, type: CreateStartScripts) {
116+
outputDir = new File(project.buildDir, 'scripts')
117+
mainClassName = mainClass
118+
applicationName = name
119+
classpath = project.tasks[JavaPlugin.JAR_TASK_NAME].outputs.files + project.configurations.runtime
120+
}
121+
project.tasks[name].dependsOn(project.jar)
122+
project.applicationDistribution.with {
123+
into("bin") {
124+
from(project.tasks[name])
125+
fileMode = 0755
126+
}
127+
}
128+
}
129+
applicationDistribution.from("../gradle/java-tron.vmoptions") {
130+
into "bin"
131+
}
132+
createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest')
133+
134+
def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true'
135+
if (releaseBinary == 'true') {
136+
artifacts {
137+
archives(binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest'))
138+
}
139+
}
140+
141+
task copyToParent(type: Copy) {
142+
into "../build/distributions"
143+
from "$buildDir/distributions"
144+
include "*.zip"
145+
}
146+
147+
148+
149+
build.finalizedBy(copyToParent)

‎plugins/src/main/java/org/tron/plugins/ArchiveManifest.java‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
package org.tron.plugins;
2+
3+
import static java.nio.charset.StandardCharsets.UTF_8;
4+
import static org.iq80.leveldb.impl.Iq80DBFactory.factory;
5+
6+
import com.beust.jcommander.JCommander;
7+
import com.beust.jcommander.Parameter;
8+
import java.io.BufferedInputStream;
9+
import java.io.File;
10+
import java.io.FileInputStream;
11+
import java.io.IOException;
12+
import java.io.InputStream;
13+
import java.nio.charset.StandardCharsets;
14+
import java.nio.file.Path;
15+
import java.nio.file.Paths;
16+
import java.util.ArrayList;
17+
import java.util.Arrays;
18+
import java.util.List;
19+
import java.util.Objects;
20+
import java.util.Properties;
21+
import java.util.concurrent.ArrayBlockingQueue;
22+
import java.util.concurrent.Callable;
23+
import java.util.concurrent.ExecutionException;
24+
import java.util.concurrent.Executors;
25+
import java.util.concurrent.Future;
26+
import java.util.concurrent.ThreadPoolExecutor;
27+
import java.util.concurrent.TimeUnit;
28+
import java.util.stream.Collectors;
29+
import lombok.extern.slf4j.Slf4j;
30+
import org.iq80.leveldb.CompressionType;
31+
import org.iq80.leveldb.DB;
32+
import org.iq80.leveldb.Options;
33+
import org.iq80.leveldb.impl.Filename;
34+
35+
@Slf4j(topic = "archive")
36+
/*
37+
a helper to rewrite leveldb manifest.
38+
*/
39+
public class ArchiveManifest implements Callable<Boolean> {
40+
41+
42+
private static final String KEY_ENGINE = "ENGINE";
43+
private static final String LEVELDB = "LEVELDB";
44+
45+
private final Path srcDbPath;
46+
private final String name;
47+
private final Options options;
48+
private final long startTime;
49+
50+
51+
private static final int CPUS = Runtime.getRuntime().availableProcessors();
52+
53+
private static final ThreadPoolExecutor EXECUTOR = new ThreadPoolExecutor(
54+
CPUS, 16 * CPUS, 1, TimeUnit.MINUTES,
55+
new ArrayBlockingQueue<>(CPUS, true), Executors.defaultThreadFactory(),
56+
new ThreadPoolExecutor.CallerRunsPolicy());
57+
58+
static {
59+
EXECUTOR.allowCoreThreadTimeOut(true);
60+
}
61+
62+
public ArchiveManifest(String src, String name, int maxManifestSize, int maxBatchSize) {
63+
this.name = name;
64+
this.srcDbPath = Paths.get(src, name);
65+
this.startTime = System.currentTimeMillis();
66+
this.options = newDefaultLevelDbOptions();
67+
this.options.maxManifestSize(maxManifestSize);
68+
this.options.maxBatchSize(maxBatchSize);
69+
}
70+
71+
@Override
72+
public Boolean call() throws Exception {
73+
return doArchive();
74+
}
75+
76+
public static org.iq80.leveldb.Options newDefaultLevelDbOptions() {
77+
org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options();
78+
dbOptions.createIfMissing(true);
79+
dbOptions.paranoidChecks(true);
80+
dbOptions.verifyChecksums(true);
81+
dbOptions.compressionType(CompressionType.SNAPPY);
82+
dbOptions.blockSize(4 * 1024);
83+
dbOptions.writeBufferSize(10 * 1024 * 1024);
84+
dbOptions.cacheSize(10 * 1024 * 1024L);
85+
dbOptions.maxOpenFiles(1000);
86+
dbOptions.maxBatchSize(64_000);
87+
dbOptions.maxManifestSize(128);
88+
dbOptions.fast(false);
89+
return dbOptions;
90+
}
91+
92+
public static void main(String[] args) {
93+
Args parameters = new Args();
94+
JCommander jc = JCommander.newBuilder()
95+
.addObject(parameters)
96+
.build();
97+
jc.parse(args);
98+
if (parameters.help) {
99+
jc.usage();
100+
return;
101+
}
102+
103+
File dbDirectory = new File(parameters.databaseDirectory);
104+
if (!dbDirectory.exists()) {
105+
logger.info("Directory {} does not exist.", parameters.databaseDirectory);
106+
return;
107+
}
108+
109+
List<File> files = Arrays.stream(Objects.requireNonNull(dbDirectory.listFiles()))
110+
.filter(File::isDirectory).collect(
111+
Collectors.toList());
112+
113+
if (files.isEmpty()) {
114+
logger.info("Directory {} does not contain any database.", parameters.databaseDirectory);
115+
return;
116+
}
117+
final long time = System.currentTimeMillis();
118+
final List<Future<Boolean>> res = new ArrayList<>();
119+
files.forEach(f -> res.add(
120+
EXECUTOR.submit(new ArchiveManifest(parameters.databaseDirectory, f.getName(),
121+
parameters.maxManifestSize, parameters.maxBatchSize))));
122+
int fails = res.size();
123+
124+
for (Future<Boolean> re : res) {
125+
try {
126+
if (Boolean.TRUE.equals(re.get())) {
127+
fails--;
128+
}
129+
} catch (InterruptedException e) {
130+
logger.error("{}", e);
131+
Thread.currentThread().interrupt();
132+
} catch (ExecutionException e) {
133+
logger.error("{}", e);
134+
}
135+
}
136+
137+
EXECUTOR.shutdown();
138+
logger.info("DatabaseDirectory:{}, maxManifestSize:{}, maxBatchSize:{},"
139+
+ "database reopen use {} seconds total.",
140+
parameters.databaseDirectory, parameters.maxManifestSize, parameters.maxBatchSize,
141+
(System.currentTimeMillis() - time) / 1000);
142+
if (fails > 0) {
143+
logger.error("Failed!!!!!!!!!!!!!!!!!!!!!!!! size:{}", fails);
144+
}
145+
System.exit(fails);
146+
}
147+
148+
public void open() throws IOException {
149+
DB database = factory.open(this.srcDbPath.toFile(), this.options);
150+
database.close();
151+
}
152+
153+
public boolean checkManifest(String dir) throws IOException {
154+
// Read "CURRENT" file, which contains a pointer to the current manifest file
155+
File currentFile = new File(dir, Filename.currentFileName());
156+
if (!currentFile.exists()) {
157+
return false;
158+
}
159+
String currentName = com.google.common.io.Files.asCharSource(currentFile, UTF_8).read();
160+
if (currentName.isEmpty() || currentName.charAt(currentName.length() - 1) != '\n') {
161+
return false;
162+
}
163+
currentName = currentName.substring(0, currentName.length() - 1);
164+
File current = new File(dir, currentName);
165+
if (!current.isFile()) {
166+
return false;
167+
}
168+
long maxSize = options.maxManifestSize();
169+
if (maxSize < 0) {
170+
return false;
171+
}
172+
logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024);
173+
if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) {
174+
logger.info("Db {} ignored.", this.name);
175+
return false;
176+
}
177+
return current.length() >= maxSize * 1024 * 1024;
178+
}
179+
180+
public boolean doArchive() throws IOException {
181+
File levelDbFile = srcDbPath.toFile();
182+
if (!levelDbFile.exists()) {
183+
logger.info("File {},does not exist, ignored.", srcDbPath.toString());
184+
return true;
185+
}
186+
if (!checkEngine()) {
187+
logger.info("Db {},not leveldb, ignored.", this.name);
188+
return true;
189+
}
190+
if (!checkManifest(levelDbFile.toString())) {
191+
logger.info("Db {},no need, ignored.", levelDbFile.toString());
192+
return true;
193+
}
194+
open();
195+
logger.info("Db {} archive use {} ms.", this.name, (System.currentTimeMillis() - startTime));
196+
return true;
197+
}
198+
199+
public boolean checkEngine() {
200+
String dir = this.srcDbPath.toString();
201+
String enginePath = dir + File.separator + "engine.properties";
202+
String engine = readProperty(enginePath, KEY_ENGINE);
203+
return LEVELDB.equals(engine);
204+
}
205+
206+
public static String readProperty(String file, String key) {
207+
try (FileInputStream fileInputStream = new FileInputStream(file);
208+
InputStream inputStream = new BufferedInputStream(fileInputStream)) {
209+
Properties prop = new Properties();
210+
prop.load(inputStream);
211+
return new String(prop.getProperty(key, "").getBytes(StandardCharsets.ISO_8859_1),
212+
UTF_8);
213+
} catch (Exception e) {
214+
logger.error("{}", e);
215+
return "";
216+
}
217+
}
218+
219+
public static class Args {
220+
@Parameter
221+
private List<String> parameters = new ArrayList<>();
222+
223+
@Parameter(names = {"-d", "--database-directory"}, description = "java-tron database directory")
224+
private String databaseDirectory = "output-directory/database";
225+
226+
@Parameter(names = {"-b", "--batch-size" }, description = "deal manifest batch size")
227+
private int maxBatchSize = 80_000;
228+
229+
@Parameter(names = {"-m", "--manifest-size" }, description = "manifest min size(M) to archive")
230+
private int maxManifestSize = 0;
231+
232+
@Parameter(names = {"-h", "--help"}, help = true)
233+
private boolean help;
234+
}
235+
}

0 commit comments

Comments
 (0)