diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9506f77bb..f00ac56e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [ '8', '11' ] + java: [ '11' ] env: PGP_KEY_PASSWORD: ${{ secrets.PGP_KEY_PASSWORD }} MAVEN_USERNAME: ${{ secrets.OSSRH_USER }} @@ -53,7 +53,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: '11' java-package: jdk architecture: x64 diff --git a/.travis.yml b/.travis.yml index 094cf0a11..056b9c803 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,5 +28,5 @@ after_success: env: global: - - NITRITE_VERSION=4.0.1-SNAPSHOT + - NITRITE_VERSION=4.1.0-SNAPSHOT - PGP_KEY_FILE=~/secring.gpg \ No newline at end of file diff --git a/README.md b/README.md index bde2283a3..952c53ebc 100644 --- a/README.md +++ b/README.md @@ -248,7 +248,7 @@ try (Session session = db.createSession()) { Migration migration1 = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instruction instructions) { + public void migrate(InstructionSet instructions) { instructions.forDatabase() // make a non-secure db to secure db .addPassword("test-user", "test-password"); @@ -288,7 +288,7 @@ Migration migration1 = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { Migration migration2 = new Migration(2, 3) { @Override - public void migrate(Instruction instructions) { + public void migrate(InstructionSet instructions) { instructions.forCollection("test") .addField("fullName", "Dummy Name"); } diff --git a/TODO.md b/TODO.md index b0fe2bdb8..bb90ca700 100644 --- a/TODO.md +++ b/TODO.md @@ -1,12 +1,6 @@ ##TODO: -1. fix codacy issues -2. nitrite explorer -4. add p2p replications via jgroups -5. add lucene indexer -6. nitrite cluster via jgroups -7. spring data rest / graphql api over nitrite cluster -8. data views (like rdbms view) + ## Other Articles @@ -158,6 +152,42 @@ https://blog.yugabyte.com/enhancing-rocksdb-for-speed-scale/ - http://www.cbcb.umd.edu/confcour/Spring2014/CMSC424/query_optimization.pdf - https://www.tutorialcup.com/dbms/query-optimization.htm - https://www.geeksforgeeks.org/query-optimization-in-relational-algebra/ + + +## TODO Test + +``` + +DatabaseInstruction dropRepository(Class type, String key) +DatabaseInstruction dropRepository(EntityDecorator decorator) +DatabaseInstruction dropRepository(EntityDecorator decorator, String key) +DatabaseInstruction dropRepository(String typeName) + + +RepositoryInstruction forRepository(Class type) +RepositoryInstruction forRepository(Class type, String key) +RepositoryInstruction forRepository(EntityDecorator decorator) +RepositoryInstruction forRepository(EntityDecorator decorator, String key) +RepositoryInstruction forRepository(String typeName) + + +EntityId + +IndexValidator + +ObjectCursor + +RepositoryFactory + + +Transaction :: ObjectRepository getRepository(EntityDecorator decorator); +Transaction :: ObjectRepository getRepository(EntityDecorator decorator, String key); + + +KNO2JacksonMapper +``` + + diff --git a/build.gradle b/build.gradle index 8912f73ac..c88137a39 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,6 @@ buildscript { repositories { - jcenter() mavenCentral() mavenLocal() maven { @@ -25,7 +24,6 @@ buildscript { } dependencies { classpath "io.freefair.gradle:lombok-plugin:6.3.0" - classpath "org.asciidoctor:asciidoctor-gradle-plugin:1.6.1" classpath "net.ltgt.gradle:gradle-errorprone-plugin:2.0.2" classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0" classpath "com.adarshr:gradle-test-logger-plugin:3.1.0" diff --git a/gradle.properties b/gradle.properties index a478be1cf..0c49d87ef 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,4 +17,7 @@ org.gradle.jvmargs=-Xmx1024m org.gradle.parallel=false #android.enableSeparateAnnotationProcessing=true # artifact version -nitriteVersion=4.0.1-SNAPSHOT +nitriteVersion=4.1.0-SNAPSHOT +android.useAndroidX=true +android.enableJetifier=true +android.suppressUnsupportedCompileSdk=32 diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/nitrite-android-example/build.gradle b/nitrite-android-example/build.gradle index fd47aeff9..874c2dc3e 100644 --- a/nitrite-android-example/build.gradle +++ b/nitrite-android-example/build.gradle @@ -17,11 +17,11 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:4.1.1' + classpath 'com.android.tools.build:gradle:7.0.4' } } @@ -29,7 +29,7 @@ apply plugin: 'com.android.application' repositories { google() - jcenter() + mavenCentral() } android { @@ -46,18 +46,16 @@ android { exclude 'META-INF/LGPL2.1' } - compileSdkVersion 28 + compileSdkVersion 32 defaultConfig { applicationId "org.dizitart.no2.example.android" minSdkVersion 19 - targetSdkVersion 28 + targetSdkVersion 32 versionCode 1 versionName "1.0" -// multiDexEnabled true - - testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner' } buildTypes { @@ -67,12 +65,7 @@ android { } } - dexOptions { - jumboMode true // required - } - compileOptions { -// coreLibraryDesugaringEnabled true sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 } @@ -87,15 +80,15 @@ dependencies { implementation project(path: ':nitrite-mvstore-adapter', configuration: 'default') implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'com.android.support:appcompat-v7:28.0.0' - implementation 'com.android.support:multidex:1.0.3' - implementation 'com.android.support.constraint:constraint-layout:2.0.4' - implementation 'com.android.support:design:28.0.0' - annotationProcessor "org.projectlombok:lombok:1.18.22" + implementation 'androidx.appcompat:appcompat:1.4.2' + implementation 'androidx.multidex:multidex:2.0.1' + implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + implementation 'com.google.android.material:material:1.6.1' + annotationProcessor 'org.projectlombok:lombok:1.18.24' testImplementation 'junit:junit:4.13.2' - testImplementation 'org.mockito:mockito-core:4.1.0' - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" - androidTestImplementation 'com.android.support.test:runner:1.0.2' - androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' + testImplementation 'org.mockito:mockito-core:4.6.1' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.24' + androidTestImplementation 'androidx.test.ext:junit:1.1.3' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0' } diff --git a/nitrite-android-example/src/androidTest/java/org/dizitart/no2/example/android/ExampleInstrumentedTest.java b/nitrite-android-example/src/androidTest/java/org/dizitart/no2/example/android/ExampleInstrumentedTest.java index d1b5ff2d6..4e6640dae 100644 --- a/nitrite-android-example/src/androidTest/java/org/dizitart/no2/example/android/ExampleInstrumentedTest.java +++ b/nitrite-android-example/src/androidTest/java/org/dizitart/no2/example/android/ExampleInstrumentedTest.java @@ -17,8 +17,8 @@ package org.dizitart.no2.example.android; import android.content.Context; -import android.support.test.InstrumentationRegistry; -import android.support.test.runner.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/nitrite-android-example/src/main/AndroidManifest.xml b/nitrite-android-example/src/main/AndroidManifest.xml index dbba5aab9..1294547f6 100644 --- a/nitrite-android-example/src/main/AndroidManifest.xml +++ b/nitrite-android-example/src/main/AndroidManifest.xml @@ -19,7 +19,7 @@ xmlns:tools="http://schemas.android.com/tools" package="org.dizitart.no2.example.android"> + android:theme="@style/AppTheme.NoActionBar" + android:exported="false"> diff --git a/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/MainActivity.java b/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/MainActivity.java index e9d850d3c..d74f1973b 100644 --- a/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/MainActivity.java +++ b/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/MainActivity.java @@ -17,7 +17,7 @@ package org.dizitart.no2.example.android; import android.os.Bundle; -import android.support.v7.app.AppCompatActivity; +import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.view.Menu; import android.view.MenuItem; @@ -26,6 +26,7 @@ import android.widget.ProgressBar; import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.events.CollectionEventListener; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.common.util.Iterables; import org.dizitart.no2.filters.Filter; import org.dizitart.no2.mvstore.MVStoreModule; @@ -55,8 +56,11 @@ protected void onCreate(Bundle savedInstanceState) { String fileName = getFilesDir().getPath() + "/test.db"; Log.i("Nitrite", "Nitrite file - " + fileName); + SimpleDocumentMapper documentMapper = new SimpleDocumentMapper(); + documentMapper.registerEntityConverter(new User.Converter()); db = Nitrite.builder() .loadModule(new MVStoreModule(fileName)) + .loadModule(() -> Iterables.setOf(documentMapper)) .openOrCreate("test-user", "test-password"); repository = db.getRepository(User.class); diff --git a/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/User.java b/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/User.java index 628ef3acf..9aa932f44 100644 --- a/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/User.java +++ b/nitrite-android-example/src/main/java/org/dizitart/no2/example/android/User.java @@ -17,13 +17,13 @@ package org.dizitart.no2.example.android; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** * @author Anindya Chatterjee. */ -public class User implements Mappable { +public class User { private String id; private String username; private String email; @@ -62,19 +62,29 @@ public void setEmail(String email) { this.email = email; } - @Override - public Document write(NitriteMapper mapper) { - Document document = Document.createDocument(); - document.put("id", id); - document.put("username", username); - document.put("email", email); - return document; - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return User.class; + } + + @Override + public Document toDocument(User entity, NitriteMapper nitriteMapper) { + Document document = Document.createDocument(); + document.put("id", entity.id); + document.put("username", entity.username); + document.put("email", entity.email); + return document; + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = (String) document.get("id"); - username = (String) document.get("username"); - email = (String) document.get("email"); + @Override + public User fromDocument(Document document, NitriteMapper nitriteMapper) { + User entity = new User(); + entity.id = (String) document.get("id"); + entity.username = (String) document.get("username"); + entity.email = (String) document.get("email"); + return entity; + } } } diff --git a/nitrite-bom/build.gradle b/nitrite-bom/build.gradle index 15a765ed5..4636b3517 100644 --- a/nitrite-bom/build.gradle +++ b/nitrite-bom/build.gradle @@ -22,20 +22,18 @@ dependencies { api project(":nitrite-rocksdb-adapter") api "org.slf4j:slf4j-api:1.7.32" - api "org.objenesis:objenesis:2.6" api "org.jasypt:jasypt:1.9.3" - api "com.fasterxml.jackson.core:jackson-databind:2.13.0" - api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.1" + api "com.fasterxml.jackson.core:jackson-databind:2.13.3" + api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.3" api "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.3" - api "com.fasterxml.jackson.module:jackson-module-kotlin:2.13.0" - api "org.mapdb:mapdb:3.0.8" - api "com.h2database:h2-mvstore:1.4.200" + api "com.fasterxml.jackson.module:jackson-module-kotlin:2.13.3" + api "com.h2database:h2-mvstore:2.1.214" api "com.squareup.okhttp3:okhttp:4.9.3" api "org.rocksdb:rocksdbjni:7.2.2" - api "com.esotericsoftware.kryo:kryo5:5.2.1" + api "com.esotericsoftware.kryo:kryo5:5.3.0" api "org.locationtech.jts:jts-core:1.18.2" api "commons-codec:commons-codec:1.15" - api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.0" + api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.21" api "org.jetbrains.kotlin:kotlin-reflect:1.6.21" } } diff --git a/nitrite-jackson-mapper/build.gradle b/nitrite-jackson-mapper/build.gradle index 29b1f918c..71a45c19d 100644 --- a/nitrite-jackson-mapper/build.gradle +++ b/nitrite-jackson-mapper/build.gradle @@ -47,18 +47,21 @@ dependencies { api "org.slf4j:slf4j-api" api "com.fasterxml.jackson.core:jackson-databind" - annotationProcessor "org.projectlombok:lombok:1.18.22" + annotationProcessor "org.projectlombok:lombok:1.18.24" testImplementation project(path: ':nitrite-mvstore-adapter', configuration: 'default') - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" + testAnnotationProcessor "org.projectlombok:lombok:1.18.24" testImplementation "junit:junit:4.13.2" testImplementation "org.locationtech.jts:jts-core:1.18.2" - testImplementation "org.awaitility:awaitility:4.1.1" - testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.14.1" - testImplementation "org.apache.logging.log4j:log4j-core:2.15.0" - testImplementation "com.github.javafaker:javafaker:1.0.2" + testImplementation "org.awaitility:awaitility:4.2.0" + testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2" + testImplementation "org.apache.logging.log4j:log4j-core:2.17.2" + testImplementation ("com.github.javafaker:javafaker:1.0.2") { + exclude module: 'snakeyaml' + } testImplementation platform("com.fasterxml.jackson:jackson-bom:2.13.0") testImplementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" + testImplementation 'org.yaml:snakeyaml:1.30' } test { diff --git a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapper.java b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapper.java index 0af159b4f..a261800e4 100644 --- a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapper.java +++ b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapper.java @@ -18,71 +18,46 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.AccessLevel; -import lombok.Getter; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.*; import lombok.extern.slf4j.Slf4j; import org.dizitart.no2.NitriteConfig; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.util.ObjectUtils; +import org.dizitart.no2.common.mapper.modules.NitriteIdModule; import org.dizitart.no2.exceptions.ObjectMappingException; -import org.dizitart.no2.common.mapper.extensions.NitriteIdExtension; import java.io.IOException; -import java.lang.reflect.Modifier; import java.util.*; -import static org.dizitart.no2.common.util.Iterables.listOf; +import static org.dizitart.no2.common.util.ValidationUtils.notNull; /** * @author Anindya Chatterjee */ @Slf4j -public class JacksonMapper extends MappableMapper { - private final List jacksonExtensions; - private final List> moduleTypes; - - @Getter(AccessLevel.PROTECTED) - private final ObjectMapper objectMapper; - - public JacksonMapper() { - this.jacksonExtensions = new ArrayList<>(); - this.moduleTypes = new ArrayList<>(); - this.objectMapper = createObjectMapper(); - } - - public JacksonMapper(JacksonExtension... jacksonExtensions) { - this.jacksonExtensions = new ArrayList<>(listOf(jacksonExtensions)); - this.moduleTypes = new ArrayList<>(); - this.objectMapper = createObjectMapper(); - } - - protected ObjectMapper createObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setVisibility( - objectMapper.getSerializationConfig().getDefaultVisibilityChecker() - .withFieldVisibility(JsonAutoDetect.Visibility.ANY) - .withGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE)); - objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); - objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); - objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - this.jacksonExtensions.add(new NitriteIdExtension()); - for (JacksonExtension jacksonExtension : jacksonExtensions) { - loadJacksonExtension(jacksonExtension, objectMapper); +public class JacksonMapper implements NitriteMapper { + private ObjectMapper objectMapper; + + protected ObjectMapper getObjectMapper() { + if (objectMapper == null) { + objectMapper = new ObjectMapper(); + objectMapper.setVisibility( + objectMapper.getSerializationConfig().getDefaultVisibilityChecker() + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE)); + objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); + objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.registerModule(new NitriteIdModule()); } return objectMapper; } - @Override - protected void addValueType(Class valueType) { - super.addValueType(valueType); - this.moduleTypes.add(valueType); + public void registerJacksonModule(Module module) { + notNull(module, "module cannot be null"); + getObjectMapper().registerModule(module); } @Override @@ -92,103 +67,61 @@ public Target convert(Source source, Class type) { return null; } - if (isValue(source)) { - if (this.moduleTypes.contains(type)) { - return this.objectMapper.convertValue(source, type); + try { + JsonNode node = getObjectMapper().convertValue(source, JsonNode.class); + if (node == null) return null; + + if (node.isValueNode()) { + return getNodeValue(node); } else { - return (Target) convertValue(source); - } - } else { - if (Document.class.isAssignableFrom(type)) { - return (Target) convertToDocument(source); - } else if (source instanceof Document) { - return convertFromDocument((Document) source, type); + if (Document.class.isAssignableFrom(type)) { + return (Target) convertToDocument(source); + } else if (source instanceof Document) { + return convertFromDocument((Document) source, type); + } } + } catch (Exception e) { + throw new ObjectMappingException("Failed to convert object of type " + + source.getClass() + " to type " + type, e); } - throw new ObjectMappingException("failed to convert using jackson"); + throw new ObjectMappingException("Can't convert object to type " + type + + ", try registering a jackson Module for it."); } - @Override - public boolean isValueType(Class type) { - if (super.isValueType(type)) return true; - if (moduleTypes.contains(type)) return true; - if (type.isInterface() || Modifier.isAbstract(type.getModifiers())) return false; - Object item = ObjectUtils.newInstance(type, false); - return isValue(item); - } - - @Override - public boolean isValue(Object object) { - try { - JsonNode node = objectMapper.convertValue(object, JsonNode.class); - return node != null && node.isValueNode(); - } catch (Exception ex) { - throw new ObjectMappingException("error while checking for value type", ex); - } - } @Override public void initialize(NitriteConfig nitriteConfig) { - } - @Override protected Target convertFromDocument(Document source, Class type) { try { - return super.convertFromDocument(source, type); - } catch (ObjectMappingException ome) { - try { - return objectMapper.convertValue(source, type); - } catch (IllegalArgumentException iae) { - log.error("Error while converting document to object ", iae); - if (iae.getCause() instanceof JsonMappingException) { - JsonMappingException jme = (JsonMappingException) iae.getCause(); - if (jme.getMessage().contains("Cannot construct instance")) { - throw new ObjectMappingException(jme.getMessage()); - } + return getObjectMapper().convertValue(source, type); + } catch (IllegalArgumentException iae) { + if (iae.getCause() instanceof JsonMappingException) { + JsonMappingException jme = (JsonMappingException) iae.getCause(); + if (jme.getMessage().contains("Cannot construct instance")) { + throw new ObjectMappingException(jme.getMessage()); } - throw iae; } + throw iae; } } - @Override protected Document convertToDocument(Source source) { - try { - return super.convertToDocument(source); - } catch (ObjectMappingException ome) { - JsonNode node = objectMapper.convertValue(source, JsonNode.class); - return readDocument(node); - } + JsonNode node = getObjectMapper().convertValue(source, JsonNode.class); + return readDocument(node); } - private void loadJacksonExtension(JacksonExtension jacksonExtension, ObjectMapper objectMapper) { - for (Class dataType : jacksonExtension.getSupportedTypes()) { - addValueType(dataType); - } - objectMapper.registerModule(jacksonExtension.getModule()); - } - - private Object convertValue(Object object) { - JsonNode node = objectMapper.convertValue(object, JsonNode.class); - if (node == null) { - return null; - } - + @SuppressWarnings("unchecked") + private T getNodeValue(JsonNode node) { switch (node.getNodeType()) { case NUMBER: - return node.numberValue(); + return (T) node.numberValue(); case STRING: - return node.textValue(); + return (T) node.textValue(); case BOOLEAN: - return node.booleanValue(); - case ARRAY: - case BINARY: - case MISSING: - case NULL: - case OBJECT: - case POJO: + return (T) Boolean.valueOf(node.booleanValue()); default: return null; } diff --git a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapperModule.java b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapperModule.java index 9fe5f54bd..7b7d6d55d 100644 --- a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapperModule.java +++ b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonMapperModule.java @@ -16,6 +16,7 @@ package org.dizitart.no2.common.mapper; +import com.fasterxml.jackson.databind.Module; import org.dizitart.no2.common.module.NitriteModule; import org.dizitart.no2.common.module.NitritePlugin; @@ -33,8 +34,11 @@ public JacksonMapperModule() { jacksonMapper = new JacksonMapper(); } - public JacksonMapperModule(JacksonExtension... jacksonExtensions) { - jacksonMapper = new JacksonMapper(jacksonExtensions); + public JacksonMapperModule(Module... jacksonModules) { + jacksonMapper = new JacksonMapper(); + for (Module jacksonModule : jacksonModules) { + jacksonMapper.registerJacksonModule(jacksonModule); + } } @Override diff --git a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdExtension.java b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdExtension.java deleted file mode 100644 index 6523a9991..000000000 --- a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdExtension.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2019-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.common.mapper.extensions; - -import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.module.SimpleModule; -import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.common.mapper.JacksonExtension; - -import java.util.List; - -import static org.dizitart.no2.common.util.Iterables.listOf; - -/** - * Class that registers capability of serializing {@code NitriteId} with the Jackson core. - * - * @author Anindya Chatterjee - * @since 1.0.0 - */ -public class NitriteIdExtension implements JacksonExtension { - - @Override - public List> getSupportedTypes() { - return listOf(NitriteId.class); - } - - @Override - public Module getModule() { - return new SimpleModule() { - @Override - public void setupModule(SetupContext context) { - addSerializer(NitriteId.class, new NitriteIdSerializer()); - addDeserializer(NitriteId.class, new NitriteIdDeserializer()); - super.setupModule(context); - } - }; - } -} diff --git a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdDeserializer.java b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdDeserializer.java similarity index 96% rename from nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdDeserializer.java rename to nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdDeserializer.java index f69a97aac..ca07c4c6f 100644 --- a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdDeserializer.java +++ b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdDeserializer.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.dizitart.no2.common.mapper.extensions; +package org.dizitart.no2.common.mapper.modules; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; diff --git a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdModule.java b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdModule.java new file mode 100644 index 000000000..2c0c451e7 --- /dev/null +++ b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdModule.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019-2020. Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dizitart.no2.common.mapper.modules; + +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.dizitart.no2.collection.NitriteId; + +/** + * Class that registers capability of serializing {@link NitriteId} with the Jackson core. + * + * @author Anindya Chatterjee + * @since 1.0.0 + */ +public class NitriteIdModule extends SimpleModule { + + @Override + public void setupModule(SetupContext context) { + addSerializer(NitriteId.class, new NitriteIdSerializer()); + addDeserializer(NitriteId.class, new NitriteIdDeserializer()); + super.setupModule(context); + } +} diff --git a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdSerializer.java b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdSerializer.java similarity index 96% rename from nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdSerializer.java rename to nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdSerializer.java index 2262db37d..1b6b34e2b 100644 --- a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/extensions/NitriteIdSerializer.java +++ b/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/modules/NitriteIdSerializer.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.dizitart.no2.common.mapper.extensions; +package org.dizitart.no2.common.mapper.modules; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/JacksonMapperTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/JacksonMapperTest.java index 72eec9f15..21ec94a0b 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/JacksonMapperTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/JacksonMapperTest.java @@ -18,8 +18,6 @@ package org.dizitart.no2.common.mapper; import com.fasterxml.jackson.databind.DeserializationConfig; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.cfg.ContextAttributes; import com.fasterxml.jackson.databind.introspect.VisibilityChecker; @@ -29,8 +27,7 @@ import com.fasterxml.jackson.databind.node.BinaryNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.dizitart.no2.NitriteConfig; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.extensions.NitriteIdExtension; +import org.dizitart.no2.common.mapper.modules.NitriteIdModule; import org.dizitart.no2.exceptions.ObjectMappingException; import org.junit.Test; @@ -80,10 +77,13 @@ public void testConstructor() { @Test public void testConstructor2() { - NitriteIdExtension nitriteIdExtension = new NitriteIdExtension(); - NitriteIdExtension nitriteIdExtension1 = new NitriteIdExtension(); - ObjectMapper objectMapper = (new JacksonMapper(nitriteIdExtension, nitriteIdExtension1, new NitriteIdExtension())) - .getObjectMapper(); + NitriteIdModule nitriteIdModule = new NitriteIdModule(); + NitriteIdModule nitriteIdModule1 = new NitriteIdModule(); + JacksonMapper jacksonMapper = new JacksonMapper(); + jacksonMapper.registerJacksonModule(nitriteIdModule); + jacksonMapper.registerJacksonModule(nitriteIdModule1); + jacksonMapper.registerJacksonModule(new NitriteIdModule()); + ObjectMapper objectMapper = jacksonMapper.getObjectMapper(); PolymorphicTypeValidator polymorphicTypeValidator = objectMapper.getPolymorphicTypeValidator(); assertTrue(polymorphicTypeValidator instanceof LaissezFaireSubTypeValidator); VisibilityChecker visibilityChecker = objectMapper.getVisibilityChecker(); @@ -120,7 +120,9 @@ public void testConstructor2() { @Test public void testConstructor3() { - ObjectMapper objectMapper = (new JacksonMapper(new NitriteIdExtension())).getObjectMapper(); + JacksonMapper jacksonMapper = new JacksonMapper(); + jacksonMapper.registerJacksonModule(new NitriteIdModule()); + ObjectMapper objectMapper = jacksonMapper.getObjectMapper(); PolymorphicTypeValidator polymorphicTypeValidator = objectMapper.getPolymorphicTypeValidator(); assertTrue(polymorphicTypeValidator instanceof LaissezFaireSubTypeValidator); VisibilityChecker visibilityChecker = objectMapper.getVisibilityChecker(); @@ -156,8 +158,8 @@ public void testConstructor3() { } @Test - public void testCreateObjectMapper() { - ObjectMapper actualCreateObjectMapperResult = (new JacksonMapper()).createObjectMapper(); + public void testGetObjectMapper() { + ObjectMapper actualCreateObjectMapperResult = (new JacksonMapper()).getObjectMapper(); PolymorphicTypeValidator polymorphicTypeValidator = actualCreateObjectMapperResult.getPolymorphicTypeValidator(); assertTrue(polymorphicTypeValidator instanceof LaissezFaireSubTypeValidator); VisibilityChecker visibilityChecker = actualCreateObjectMapperResult.getVisibilityChecker(); @@ -202,15 +204,6 @@ public void testConvert() { .getSerializerProviderInstance() instanceof com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.Impl); } - @Test - public void testConvert2() { - JacksonMapper jacksonMapper = new JacksonMapper(); - jacksonMapper.addValueType(Object.class); - assertEquals("Source", jacksonMapper.convert("Source", Object.class)); - assertTrue(jacksonMapper.getObjectMapper() - .getSerializerProviderInstance() instanceof com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.Impl); - } - @Test public void testConvert3() { JacksonMapper jacksonMapper = new JacksonMapper(); @@ -239,53 +232,6 @@ public void testConvert6() throws UnsupportedEncodingException { assertNull(jacksonMapper.convert(source, Object.class)); } - @Test - public void testIsValueType() { - JacksonMapper jacksonMapper = new JacksonMapper(); - assertThrows(ObjectMappingException.class, () -> jacksonMapper.isValueType(Object.class)); - } - - @Test - public void testIsValueType2() { - JacksonMapper jacksonMapper = new JacksonMapper(); - jacksonMapper.addValueType(Object.class); - assertTrue(jacksonMapper.isValueType(Object.class)); - } - - @Test - public void testIsValueType3() { - JacksonMapper jacksonMapper = new JacksonMapper(); - assertFalse(jacksonMapper.isValueType(JsonMappingException.class)); - assertTrue(jacksonMapper.getObjectMapper() - .getSerializerProviderInstance() instanceof com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.Impl); - } - - @Test - public void testIsValueType4() { - JacksonMapper jacksonMapper = new JacksonMapper(); - assertFalse(jacksonMapper.isValueType(JsonNode.class)); - } - - @Test - public void testIsValueType5() { - JacksonMapper jacksonMapper = new JacksonMapper(); - assertFalse(jacksonMapper.isValueType(Document.class)); - } - - @Test - public void testIsValue() { - JacksonMapper jacksonMapper = new JacksonMapper(); - assertTrue(jacksonMapper.isValue("Object")); - assertTrue(jacksonMapper.getObjectMapper() - .getSerializerProviderInstance() instanceof com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.Impl); - } - - @Test - public void testIsValue2() { - JacksonMapper jacksonMapper = new JacksonMapper(); - assertFalse(jacksonMapper.isValue(new ArrayNode(new JsonNodeFactory(true)))); - } - @Test public void testInitialize() { JacksonMapper jacksonMapper = new JacksonMapper(); diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdDeserializerTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/modules/NitriteIdDeserializerTest.java similarity index 95% rename from nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdDeserializerTest.java rename to nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/modules/NitriteIdDeserializerTest.java index 9aceea44e..7528e79a1 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdDeserializerTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/modules/NitriteIdDeserializerTest.java @@ -15,7 +15,7 @@ * */ -package org.dizitart.no2.common.mapper.extensions; +package org.dizitart.no2.common.mapper.modules; import org.dizitart.no2.collection.NitriteId; import org.junit.Test; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdSerializerTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/modules/NitriteIdSerializerTest.java similarity index 95% rename from nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdSerializerTest.java rename to nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/modules/NitriteIdSerializerTest.java index a1b7f61eb..54030b1ff 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdSerializerTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/modules/NitriteIdSerializerTest.java @@ -15,7 +15,7 @@ * */ -package org.dizitart.no2.common.mapper.extensions; +package org.dizitart.no2.common.mapper.modules; import org.junit.Test; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/utils/ObjectUtilsTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/utils/ObjectUtilsTest.java new file mode 100644 index 000000000..ae62377b6 --- /dev/null +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/utils/ObjectUtilsTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.common.utils; + +import org.dizitart.no2.common.mapper.JacksonMapper; +import org.dizitart.no2.integration.repository.data.Book; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.URI; +import java.net.URL; +import java.util.*; +import java.util.regex.Pattern; + +import static org.dizitart.no2.common.util.ObjectUtils.newInstance; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class ObjectUtilsTest { + @Test + public void testNewInstance() { + JacksonMapper mapper = new JacksonMapper(); + + assertNotNull(newInstance(Object.class, false, mapper)); + assertNotNull(newInstance(Book.class, false, mapper)); + + assertNull(newInstance(byte[].class, false, mapper)); + assertNull(newInstance(Number.class, false, mapper)); + assertNull(newInstance(Byte.class, false, mapper)); + assertNull(newInstance(Short.class, false, mapper)); + assertNull(newInstance(Integer.class, false, mapper)); + assertNull(newInstance(Long.class, false, mapper)); + assertNull(newInstance(Float.class, false, mapper)); + assertNull(newInstance(Double.class, false, mapper)); + assertNull(newInstance(BigDecimal.class, false, mapper)); + assertNull(newInstance(BigInteger.class, false, mapper)); + assertNull(newInstance(Boolean.class, false, mapper)); + assertNull(newInstance(Character.class, false, mapper)); + assertNull(newInstance(String.class, false, mapper)); + assertNull(newInstance(Date.class, false, mapper)); + assertNull(newInstance(URL.class, false, mapper)); + assertNull(newInstance(URI.class, false, mapper)); + assertNull(newInstance(Currency.class, false, mapper)); + assertNull(newInstance(Calendar.class, false, mapper)); + assertNull(newInstance(StringBuffer.class, false, mapper)); + assertNull(newInstance(StringBuilder.class, false, mapper)); + assertNull(newInstance(Locale.class, false, mapper)); + assertNull(newInstance(Void.class, false, mapper)); + assertNull(newInstance(UUID.class, false, mapper)); + assertNull(newInstance(Pattern.class, false, mapper)); + + assertNull(newInstance(GregorianCalendar.class, false, mapper)); + } +} diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/NitriteTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/NitriteTest.java new file mode 100644 index 000000000..2f86eafda --- /dev/null +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/NitriteTest.java @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.dizitart.no2.Nitrite; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.common.mapper.JacksonMapperModule; +import org.dizitart.no2.exceptions.NitriteIOException; +import org.dizitart.no2.exceptions.ValidationException; +import org.dizitart.no2.index.IndexOptions; +import org.dizitart.no2.index.IndexType; +import org.dizitart.no2.integration.repository.Retry; +import org.dizitart.no2.repository.ObjectRepository; +import org.dizitart.no2.repository.annotations.Id; +import org.dizitart.no2.repository.annotations.Index; +import org.dizitart.no2.repository.annotations.Indices; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Locale; +import java.util.Set; + +import static org.dizitart.no2.collection.Document.createDocument; +import static org.dizitart.no2.common.Constants.INTERNAL_NAME_SEPARATOR; +import static org.dizitart.no2.common.Constants.META_MAP_NAME; +import static org.dizitart.no2.filters.Filter.ALL; +import static org.dizitart.no2.integration.repository.TestUtil.createDb; +import static org.junit.Assert.*; + +/** + * @author Anindya Chatterjee. + */ +public class NitriteTest { + @Rule + public Retry retry = new Retry(3); + private Nitrite db; + private NitriteCollection collection; + + @Before + public void setUp() throws ParseException { + db = createDb(new JacksonMapperModule()); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH); + + Document doc1 = createDocument("firstName", "fn1") + .put("lastName", "ln1") + .put("birthDay", simpleDateFormat.parse("2012-07-01T16:02:48.440Z")) + .put("data", new byte[]{1, 2, 3}) + .put("body", "a quick brown fox jump over the lazy dog"); + Document doc2 = createDocument("firstName", "fn2") + .put("lastName", "ln2") + .put("birthDay", simpleDateFormat.parse("2010-06-12T16:02:48.440Z")) + .put("data", new byte[]{3, 4, 3}) + .put("body", "hello world from nitrite"); + Document doc3 = createDocument("firstName", "fn3") + .put("lastName", "ln2") + .put("birthDay", simpleDateFormat.parse("2014-04-17T16:02:48.440Z")) + .put("data", new byte[]{9, 4, 8}) + .put("body", "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + + "Sed nunc mi, mattis ullamcorper dignissim vitae, condimentum non lorem."); + + collection = db.getCollection("test"); + collection.remove(ALL); + + collection.createIndex(IndexOptions.indexOptions(IndexType.FULL_TEXT), "body"); + collection.createIndex(IndexOptions.indexOptions(IndexType.UNIQUE), "firstName"); + collection.insert(doc1, doc2, doc3); + } + + @After + public void tearDown() { + if (collection.isOpen()) { + collection.remove(ALL); + collection.close(); + } + if (db != null && !db.isClosed()) { + try { + db.close(); + } catch (NitriteIOException ignore) { + } + } + } + + @Test + public void testListCollectionNames() { + Set collectionNames = db.listCollectionNames(); + assertEquals(collectionNames.size(), 1); + } + + @Test(expected = ValidationException.class) + public void testListRepositories() { + db.getRepository(EmptyClass.class); + } + + @Test + public void testListRepositories2() { + db.getRepository(Receipt.class); + Set repositories = db.listRepositories(); + assertEquals(repositories.size(), 1); + } + + @Test + public void testHasCollection() { + assertTrue(db.hasCollection("test")); + assertFalse(db.hasCollection("lucene" + INTERNAL_NAME_SEPARATOR + "test")); + } + + @Test(expected = ValidationException.class) + public void testHasRepository() { + db.getRepository(EmptyClass.class); + } + + @Test + public void testHasRepository2() { + db.getRepository(Receipt.class); + assertTrue(db.hasRepository(Receipt.class)); + assertFalse(db.hasRepository(String.class)); + } + + @Test + public void testClose() { + NitriteCollection testCollection = db.getCollection("test"); + testCollection.insert(createDocument("a", "b")); + db.close(); + + assertFalse(testCollection.isOpen()); + } + + @Test + public void testGetCollection() { + NitriteCollection collection = db.getCollection("test-collection"); + assertNotNull(collection); + assertEquals(collection.getName(), "test-collection"); + } + + @Test(expected = ValidationException.class) + public void testGetRepository() { + ObjectRepository repository = db.getRepository(EmptyClass.class); + } + + @Test + public void testGetRepository2() { + ObjectRepository repository = db.getRepository(Receipt.class); + assertNotNull(repository); + assertEquals(repository.getType(), Receipt.class); + } + + @Test(expected = ValidationException.class) + public void testGetRepositoryWithKey() { + ObjectRepository repository = db.getRepository(EmptyClass.class, "key"); + } + + @Test + public void testGetRepositoryWithKey2() { + ObjectRepository repository = db.getRepository(Receipt.class, "key"); + assertNotNull(repository); + assertEquals(repository.getType(), Receipt.class); + assertFalse(db.hasRepository(Receipt.class)); + assertTrue(db.hasRepository(Receipt.class, "key")); + } + + @Test + public void testMultipleGetCollection() { + NitriteCollection collection = db.getCollection("test-collection"); + assertNotNull(collection); + assertEquals(collection.getName(), "test-collection"); + + NitriteCollection collection2 = db.getCollection("test-collection"); + assertNotNull(collection2); + assertEquals(collection2.getName(), "test-collection"); + } + + @Test + public void testMultipleGetRepository() { + ObjectRepository repository = db.getRepository(Receipt.class); + assertNotNull(repository); + assertEquals(repository.getType(), Receipt.class); + + ObjectRepository repository2 = db.getRepository(Receipt.class); + assertNotNull(repository2); + assertEquals(repository2.getType(), Receipt.class); + } + + @Test(expected = ValidationException.class) + public void testGetRepositoryInvalid() { + db.getRepository((Class) null); + } + + @Test(expected = NitriteIOException.class) + public void testGetCollectionNullStore() { + db = Nitrite.builder().openOrCreate(); + db.close(); + db.getCollection("test"); + } + + @Test(expected = NitriteIOException.class) + public void testGetRepositoryNullStore() { + db = Nitrite.builder().openOrCreate(); + db.close(); + db.getRepository(NitriteTest.class); + } + + @Test(expected = NitriteIOException.class) + public void testGetKeyedRepositoryNullStore() { + db = Nitrite.builder().openOrCreate(); + db.close(); + db.getRepository(NitriteTest.class, "key"); + } + + @Test(expected = NitriteIOException.class) + public void testCommitNullStore() { + db = Nitrite.builder().openOrCreate(); + db.close(); + db.commit(); + } + + @Test(expected = ValidationException.class) + public void testGetCollectionInvalidName() { + db.getCollection(META_MAP_NAME); + } + + + @Data + @NoArgsConstructor + @AllArgsConstructor + @Indices({ + @Index(fields = "synced", type = IndexType.NON_UNIQUE) + }) + public static class Receipt { + @Id + private String clientRef; + private Boolean synced; + private Long createdTimestamp = System.currentTimeMillis(); + private Status status; + + public enum Status { + COMPLETED, + PREPARING, + } + } + + public static class EmptyClass { + } +} diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java index 16b83e0df..f0920e5c6 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java @@ -26,7 +26,7 @@ import org.dizitart.no2.common.mapper.JacksonMapperModule; import org.dizitart.no2.exceptions.MigrationException; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.migration.Instructions; +import org.dizitart.no2.migration.InstructionSet; import org.dizitart.no2.migration.Migration; import org.dizitart.no2.migration.TypeConverter; import org.dizitart.no2.mvstore.MVStoreModule; @@ -96,7 +96,7 @@ public void testRepositoryMigrate() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forDatabase() .addPassword("test-user", "test-password"); @@ -121,6 +121,7 @@ public void migrate(Instructions instruction) { db = Nitrite.builder() .loadModule(storeModule) + .loadModule(new JacksonMapperModule()) .fieldSeparator(".") .schemaVersion(2) .addMigrations(migration) @@ -129,7 +130,7 @@ public void migrate(Instructions instruction) { ObjectRepository newRepo = db.getRepository(NewClass.class); assertEquals(newRepo.size(), 10); assertTrue(db.listCollectionNames().isEmpty()); - assertTrue(db.listKeyedRepository().isEmpty()); + assertTrue(db.listKeyedRepositories().isEmpty()); assertEquals((int) db.getDatabaseMetaData().getSchemaVersion(), 2); } @@ -153,7 +154,7 @@ public void testCollectionMigrate() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forDatabase() .addPassword("test-user", "test-password"); @@ -184,11 +185,11 @@ public void migrate(Instructions instruction) { migration = new Migration(2, 3) { @Override - public void migrate(Instructions instructions) { - instructions.forDatabase() + public void migrate(InstructionSet instructionSet) { + instructionSet.forDatabase() .changePassword("test-user", "test-password", "password"); - instructions.forCollection("testCollectionMigrate") + instructionSet.forCollection("testCollectionMigrate") .dropIndex("firstName") .deleteField("bloodGroup") .addField("name", document -> faker.name().fullName()) @@ -235,7 +236,7 @@ public void testOpenWithoutSchemaVersion() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testOpenWithoutSchemaVersion") @@ -287,7 +288,7 @@ public void testDescendingSchema() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testDescendingSchema") @@ -313,9 +314,9 @@ public void migrate(Instructions instruction) { migration = new Migration(2, Constants.INITIAL_SCHEMA_VERSION) { @Override - public void migrate(Instructions instructions) { + public void migrate(InstructionSet instructionSet) { - instructions.forCollection("testDescendingSchema") + instructionSet.forCollection("testDescendingSchema") .rename("test"); } }; @@ -351,7 +352,7 @@ public void testMigrationWithoutVersion() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testMigrationWithoutVersion") @@ -392,7 +393,7 @@ public void testWrongSchemaVersionNoMigration() { Migration migration = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testWrongSchemaVersionNoMigration") .rename("test") @@ -421,8 +422,8 @@ public void migrate(Instructions instruction) { migration = new Migration(2, 3) { @Override - public void migrate(Instructions instructions) { - instructions.forCollection("test") + public void migrate(InstructionSet instructionSet) { + instructionSet.forCollection("test") .rename("testWrongSchemaVersionNoMigration"); } }; @@ -461,7 +462,7 @@ public void testReOpenAfterMigration() { Migration migration = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testReOpenAfterMigration") .rename("test") @@ -529,7 +530,7 @@ public void testMultipleMigrations() { Migration migration1 = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testMultipleMigrations") .rename("test"); @@ -538,7 +539,7 @@ public void migrate(Instructions instruction) { Migration migration2 = new Migration(2, 3) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .addField("fullName", "Dummy Name"); } @@ -563,7 +564,7 @@ public void migrate(Instructions instruction) { Migration migration3 = new Migration(3, 4) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .addField("age", 10); } diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java index 28f0f15db..d5cae58ad 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java @@ -28,9 +28,9 @@ */ @Data @Entity(value = "new", indices = { - @Index(value = "familyName", type = IndexType.NON_UNIQUE), - @Index(value = "fullName", type = IndexType.NON_UNIQUE), - @Index(value = "literature.ratings", type = IndexType.NON_UNIQUE), + @Index(fields = "familyName", type = IndexType.NON_UNIQUE), + @Index(fields = "fullName", type = IndexType.NON_UNIQUE), + @Index(fields = "literature.ratings", type = IndexType.NON_UNIQUE), }) public class NewClass { @Id diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java index 32a687c53..82044f93a 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java @@ -28,10 +28,10 @@ */ @Data @Entity(value = "old", indices = { - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "lastName", type = IndexType.NON_UNIQUE), - @Index(value = "literature.text", type = IndexType.FULL_TEXT), - @Index(value = "literature.ratings", type = IndexType.NON_UNIQUE), + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "lastName", type = IndexType.NON_UNIQUE), + @Index(fields = "literature.text", type = IndexType.FULL_TEXT), + @Index(fields = "literature.ratings", type = IndexType.NON_UNIQUE), }) public class OldClass { @Id diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java index 5a60bb6ec..760b19067 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java @@ -117,9 +117,9 @@ public void testFindByEmbeddedField() { @ToString @EqualsAndHashCode @Indices({ - @Index(value = "joinDate", type = IndexType.NON_UNIQUE), - @Index(value = "address", type = IndexType.FULL_TEXT), - @Index(value = "employeeNote:text", type = IndexType.FULL_TEXT) + @Index(fields = "joinDate", type = IndexType.NON_UNIQUE), + @Index(fields = "address", type = IndexType.FULL_TEXT), + @Index(fields = "employeeNote:text", type = IndexType.FULL_TEXT) }) public static class EmployeeForCustomSeparator implements Serializable { @Id diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java index 397876740..ad53f1c91 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java @@ -63,6 +63,10 @@ public void setUp() { persons.insert(person); + // process existing data + fieldProcessor.process(persons); + + // add for further changes persons.addProcessor(fieldProcessor); person = new EncryptedPerson(); @@ -187,21 +191,4 @@ public void testIndexOnEncryptedField() { EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull(); assertNull(person); } - - @Test - public void testRemoveProcessor() { - EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull(); - assertNull(person); - - person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull(); - assertNull(person); - - persons.removeProcessor(fieldProcessor); - - person = persons.find(where("cvv").eq("008")).firstOrNull(); - assertNotNull(person); - - person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull(); - assertNotNull(person); - } } diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/JacksonModuleTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/JacksonModuleTest.java index 700f007af..b392f06e5 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/JacksonModuleTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/JacksonModuleTest.java @@ -1,11 +1,9 @@ package org.dizitart.no2.integration.repository; -import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import lombok.Data; import org.dizitart.no2.Nitrite; import org.dizitart.no2.NitriteBuilder; -import org.dizitart.no2.common.mapper.JacksonExtension; import org.dizitart.no2.common.mapper.JacksonMapperModule; import org.dizitart.no2.exceptions.ObjectMappingException; import org.dizitart.no2.mvstore.MVStoreModule; @@ -18,10 +16,8 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.LocalDateTime; -import java.util.List; import java.util.UUID; -import static org.dizitart.no2.common.util.Iterables.listOf; import static org.dizitart.no2.integration.repository.BaseObjectRepositoryTest.getRandomTempDbFile; import static org.junit.Assert.assertEquals; @@ -75,7 +71,7 @@ public void testJavaTimeModule() { .build(); NitriteBuilder nitriteBuilder = Nitrite.builder() - .loadModule(new JacksonMapperModule(new JavaTimeExtension())) + .loadModule(new JacksonMapperModule(new JavaTimeModule())) .fieldSeparator(".") .loadModule(storeModule); @@ -102,17 +98,4 @@ private static class TestData { private LocalDateTime localDateTime; private Duration duration; } - - public static class JavaTimeExtension implements JacksonExtension { - - @Override - public List> getSupportedTypes() { - return listOf(LocalDateTime.class, Duration.class); - } - - @Override - public Module getModule() { - return new JavaTimeModule(); - } - } } diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java index a7a42681c..4296bcdcf 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java @@ -94,7 +94,7 @@ public void testNitriteIdField() { } @Test(expected = InvalidIdException.class) - public void setIdDuringInsert() { + public void testSetIdDuringInsert() { WithNitriteId item1 = new WithNitriteId(); item1.name = "first"; item1.idField = NitriteId.newId(); @@ -103,7 +103,7 @@ public void setIdDuringInsert() { } @Test - public void changeIdDuringUpdate() { + public void testChangeIdDuringUpdate() { WithNitriteId item2 = new WithNitriteId(); item2.name = "second"; WriteResult result = repo.insert(item2); diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectCursorTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectCursorTest.java index 132746f0e..4f9aa507c 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectCursorTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectCursorTest.java @@ -19,8 +19,8 @@ import org.dizitart.no2.common.RecordStream; import org.dizitart.no2.exceptions.ValidationException; -import org.dizitart.no2.repository.Cursor; import org.dizitart.no2.integration.repository.data.Employee; +import org.dizitart.no2.repository.Cursor; import org.junit.Test; import java.util.AbstractCollection; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java index 48307a3bd..178d688ee 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java @@ -76,7 +76,7 @@ public void testWithCircularReference() { assertEquals(instance.getParent().getName(), object.getParent().getName()); } - @Test(expected = ObjectMappingException.class) + @Test(expected = ValidationException.class) public void testWithCustomConstructor() { ObjectRepository repository = db.getRepository(WithCustomConstructor.class); diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java index 85af75e2a..e0a510606 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java @@ -21,7 +21,7 @@ import lombok.Data; import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.common.mapper.JacksonMapperModule; import org.dizitart.no2.exceptions.ValidationException; import org.dizitart.no2.index.IndexType; @@ -277,7 +277,7 @@ public void testKeyedRepository() { assertTrue(db.hasRepository(Employee.class, "developers")); assertEquals(db.listRepositories().size(), 1); - assertEquals(db.listKeyedRepository().size(), 2); + assertEquals(db.listKeyedRepositories().size(), 2); assertEquals(employeeRepo.find(where("address").text("abcd")).size(), 1); assertEquals(employeeRepo.find(where("address").text("xyz")).size(), 1); @@ -306,7 +306,7 @@ public void testEntityRepository() { assertTrue(errored); assertTrue(db.listRepositories().contains("entity.employee")); - assertEquals(db.listKeyedRepository().size(), 2); + assertEquals(db.listKeyedRepositories().size(), 2); assertEquals(db.listCollectionNames().size(), 0); assertTrue(managerRepo.hasIndex("firstName")); @@ -315,7 +315,7 @@ public void testEntityRepository() { assertTrue(employeeRepo.hasIndex("lastName")); managerRepo.drop(); - assertEquals(db.listKeyedRepository().size(), 1); + assertEquals(db.listKeyedRepositories().size(), 1); } @Test @@ -331,8 +331,8 @@ public void testIssue217() { @Data @Entity(value = "entity.employee", indices = { - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "lastName", type = IndexType.NON_UNIQUE), + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "lastName", type = IndexType.NON_UNIQUE), }) private static class EmployeeEntity { private static final Faker faker = new Faker(); diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java index 2de2e8915..dcf439b54 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java @@ -20,7 +20,7 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.*; import org.dizitart.no2.collection.events.CollectionEventListener; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.common.WriteResult; import org.dizitart.no2.common.concurrent.LockService; import org.dizitart.no2.common.mapper.JacksonMapper; @@ -63,7 +63,7 @@ public void testNullType() { RepositoryFactory factory = new RepositoryFactory(new CollectionFactory(new LockService())); JacksonMapper mapper = new JacksonMapper(); db = TestUtil.createDb(fileName, NitriteModule.module(mapper)); - factory.getRepository(db.getConfig(), null, "dummy"); + factory.getRepository(db.getConfig(), (Class) null, "dummy"); } @Test @@ -134,11 +134,6 @@ public void addProcessor(Processor processor) { } - @Override - public void removeProcessor(Processor processor) { - - } - @Override public void createIndex(IndexOptions indexOptions, String... fields) { diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java index 26c89c680..8fa0d1c98 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java @@ -39,6 +39,7 @@ import static org.awaitility.Awaitility.await; import static org.dizitart.no2.collection.Document.createDocument; +import static org.dizitart.no2.collection.UpdateOptions.updateOptions; import static org.dizitart.no2.filters.FluentFilter.where; import static org.junit.Assert.*; @@ -208,7 +209,7 @@ public void testUpsertTrue() { employee.setEmployeeNote(empNote1); WriteResult writeResult - = employeeRepository.update(where("empId").eq(12), employee, true); + = employeeRepository.update(where("empId").eq(12), employee, updateOptions(true)); assertEquals(writeResult.getAffectedCount(), 1); result = employeeRepository.find(where("joinDate").eq(joiningDate)); @@ -233,7 +234,7 @@ public void testUpsertFalse() { employee.setEmployeeNote(empNote1); WriteResult writeResult - = employeeRepository.update(where("empId").eq(12), employee, false); + = employeeRepository.update(where("empId").eq(12), employee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 0); result = employeeRepository.find(where("joinDate").eq(joiningDate)); @@ -325,7 +326,7 @@ public void testMultiUpdateWithObject() { update.setAddress("new address"); WriteResult writeResult - = employeeRepository.update(where("joinDate").eq(now), update, false); + = employeeRepository.update(where("joinDate").eq(now), update, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 0); } @@ -359,7 +360,7 @@ public void testUpdateWithChangedId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); assertEquals(count, employeeRepository.size()); @@ -378,7 +379,7 @@ public void testUpdateWithNullId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); } @@ -394,7 +395,7 @@ public void testUpdateWithDuplicateId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); assertEquals(count, employeeRepository.size()); diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java index d0bb00a1c..df689f155 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java @@ -17,7 +17,6 @@ package org.dizitart.no2.integration.repository; -import lombok.Getter; import org.dizitart.no2.common.SortOrder; import org.dizitart.no2.exceptions.FilterException; import org.dizitart.no2.exceptions.InvalidIdException; @@ -544,15 +543,6 @@ public void testIdSet() { @Test public void testBetweenFilter() { - @Getter - class TestData { - private final Date age; - - public TestData(Date age) { - this.age = age; - } - } - TestData data1 = new TestData(new GregorianCalendar(2020, Calendar.JANUARY, 11).getTime()); TestData data2 = new TestData(new GregorianCalendar(2021, Calendar.FEBRUARY, 12).getTime()); TestData data3 = new TestData(new GregorianCalendar(2022, Calendar.MARCH, 13).getTime()); diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java index fac81c611..566dc82b0 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java @@ -181,7 +181,7 @@ public void testUniversalFullTextIndexing() { } @Indices( - @Index(value = "text", type = IndexType.FULL_TEXT) + @Index(fields = "text", type = IndexType.FULL_TEXT) ) public static class TextData { public int id; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Book.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Book.java index 5bffe69e8..e14af920b 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Book.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Book.java @@ -31,13 +31,13 @@ */ @Data @Entity(value = "books", indices = { - @Index(value = "tags", type = IndexType.NON_UNIQUE), - @Index(value = "description", type = IndexType.FULL_TEXT), - @Index(value = { "price", "publisher" }) + @Index(fields = "tags", type = IndexType.NON_UNIQUE), + @Index(fields = "description", type = IndexType.FULL_TEXT), + @Index(fields = { "price", "publisher" }) }) public class Book { @JsonProperty("book_id") - @Id(fieldName = "book_id") + @Id(fieldName = "book_id", embeddedFields = { "isbn", "book_name" }) private BookId bookId; private String publisher; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java index 988908919..b051dfe18 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java @@ -19,17 +19,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; -import org.dizitart.no2.repository.annotations.Embedded; /** * @author Anindya Chatterjee */ @Data public class BookId { - @Embedded(order = 0) private String isbn; - @Embedded(order = 1, fieldName = "book_name") @JsonProperty("book_name") private String name; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Company.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Company.java index 4a85b5156..3a6721f73 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Company.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Company.java @@ -39,7 +39,7 @@ @ToString @EqualsAndHashCode @Indices({ - @Index(value = "companyName") + @Index(fields = "companyName") }) public class Company implements Serializable { @Id(fieldName = "company_id") diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java index ed048f2e6..fa84714f9 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java @@ -33,9 +33,9 @@ */ @ToString @EqualsAndHashCode -@Index(value = "joinDate", type = IndexType.NON_UNIQUE) -@Index(value = "address", type = IndexType.FULL_TEXT) -@Index(value = "employeeNote.text", type = IndexType.FULL_TEXT) +@Index(fields = "joinDate", type = IndexType.NON_UNIQUE) +@Index(fields = "address", type = IndexType.FULL_TEXT) +@Index(fields = "employeeNote.text", type = IndexType.FULL_TEXT) public class Employee implements Serializable { @Id @Getter diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Note.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Note.java index 647776b0f..68942f3c3 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Note.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/Note.java @@ -20,9 +20,6 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; import java.io.Serializable; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java index 844e39223..f27a3f869 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java @@ -29,7 +29,7 @@ */ @Getter @Setter -@Index(value = "date") +@Index(fields = "date") public class ParentClass extends SuperDuperClass { @Id protected Long id; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java index cc871a260..cf6d420fa 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java @@ -31,8 +31,8 @@ */ @Data @Entity(value = "MyPerson", indices = { - @Index(value = "name", type = IndexType.FULL_TEXT), - @Index(value = "status", type = IndexType.NON_UNIQUE) + @Index(fields = "name", type = IndexType.FULL_TEXT), + @Index(fields = "status", type = IndexType.NON_UNIQUE) }) public class PersonEntity { @Id diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java index a56a63b56..5d5423a1b 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java @@ -25,9 +25,9 @@ * @author Anindya Chatterjee */ @Data -@Index(value = "firstName") -@Index(value = "age", type = IndexType.NON_UNIQUE) -@Index(value = "lastName", type = IndexType.FULL_TEXT) +@Index(fields = "firstName") +@Index(fields = "age", type = IndexType.NON_UNIQUE) +@Index(fields = "lastName", type = IndexType.FULL_TEXT) public class RepeatableIndexTest { private String firstName; private Integer age; diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java index d39ddf146..523cebb87 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java @@ -27,7 +27,7 @@ */ @Getter @Setter -@Index(value = "text", type = IndexType.FULL_TEXT) +@Index(fields = "text", type = IndexType.FULL_TEXT) public class SuperDuperClass { private String text; } diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/TestData.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/TestData.java new file mode 100644 index 000000000..d4a1733f7 --- /dev/null +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/repository/data/TestData.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.data; + +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.Date; + +@Getter +@NoArgsConstructor +public class TestData { + private Date age; + + public TestData(Date age) { + this.age = age; + } +} diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java index 50af4f042..d2da4fd6d 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java +++ b/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java @@ -20,7 +20,7 @@ import com.github.javafaker.Faker; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.TransactionException; import org.dizitart.no2.index.IndexType; @@ -324,11 +324,10 @@ public void testRollbackClear() { repository.insert(txData2); transaction.commit(); - fail(); } catch (TransactionException e) { assert transaction != null; transaction.rollback(); - assertEquals(2, repository.size()); + assertEquals(0, repository.size()); } } } @@ -454,7 +453,7 @@ public void testCommitDropRepository() { boolean expectedException = false; try { assertEquals(0, txRepo.size()); - } catch (NitriteIOException e) { + } catch (TransactionException e) { expectedException = true; } assertTrue(expectedException); diff --git a/nitrite-mvstore-adapter/build.gradle b/nitrite-mvstore-adapter/build.gradle index 9d14111d0..073084468 100644 --- a/nitrite-mvstore-adapter/build.gradle +++ b/nitrite-mvstore-adapter/build.gradle @@ -46,26 +46,28 @@ dependencies { api project(':nitrite') api "org.slf4j:slf4j-api" api "com.h2database:h2-mvstore" - annotationProcessor "org.projectlombok:lombok:1.18.22" + annotationProcessor "org.projectlombok:lombok:1.18.24" - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" - testImplementation "uk.co.jemos.podam:podam:7.2.7.RELEASE" - testImplementation "com.github.javafaker:javafaker:1.0.2" + testAnnotationProcessor "org.projectlombok:lombok:1.18.24" + testImplementation "uk.co.jemos.podam:podam:7.2.9.RELEASE" + testImplementation ("com.github.javafaker:javafaker:1.0.2") { + exclude module: 'snakeyaml' + } testImplementation "junit:junit:4.13.2" - testImplementation "org.mockito:mockito-core:4.1.0" - testImplementation "org.apache.lucene:lucene-core:8.11.0" - testImplementation "org.apache.lucene:lucene-analyzers-common:8.11.0" - testImplementation "org.apache.lucene:lucene-queryparser:8.11.0" - testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.14.1" - testImplementation "org.apache.logging.log4j:log4j-core:2.15.0" - testImplementation "org.awaitility:awaitility:4.1.1" - testImplementation "joda-time:joda-time:2.10.13" + testImplementation "org.mockito:mockito-core:4.6.1" + testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2" + testImplementation "org.apache.logging.log4j:log4j-core:2.17.2" + testImplementation "org.awaitility:awaitility:4.2.0" + testImplementation "joda-time:joda-time:2.10.14" testImplementation "org.meanbean:meanbean:2.0.3" - testImplementation "com.fasterxml.jackson.core:jackson-databind:2.13.0" + testImplementation ("com.fasterxml.jackson.core:jackson-databind:2.13.3") { + exclude module: 'org.yaml' + } testImplementation "commons-io:commons-io:2.11.0" - testImplementation "com.google.guava:guava:31.0.1-jre" - testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:3.0.1" - testImplementation "com.sun.xml.bind:jaxb-impl:3.0.2" + testImplementation 'com.google.guava:guava:31.1-jre' + testImplementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0' + testImplementation 'com.sun.xml.bind:jaxb-impl:4.0.0' + testImplementation 'org.yaml:snakeyaml:1.30' } test { diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVSpatialKey.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVSpatialKey.java new file mode 100644 index 000000000..c7fc552b1 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVSpatialKey.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.mvstore; + +import org.dizitart.no2.common.util.SpatialKey; +import org.h2.mvstore.rtree.Spatial; + +import java.util.Arrays; + +/** + * @author Anindya Chatterjee + */ +public class MVSpatialKey extends SpatialKey implements Spatial { + private final float[] minMax; + public MVSpatialKey(long id, float... minMax) { + super(id, minMax); + this.minMax = minMax; + } + + @Override + public Spatial clone(long id) { + return new MVSpatialKey(id, this.minMax.clone()); + } + + @Override + public boolean equalsIgnoringId(Spatial o) { + return Arrays.equals(minMax, ((MVSpatialKey)o).minMax); + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreConfig.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreConfig.java index 310a5ac0a..1a2d6bc66 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreConfig.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreConfig.java @@ -86,4 +86,23 @@ public class MVStoreConfig implements StoreConfig { public void addStoreEventListener(StoreEventListener listener) { eventListeners.add(listener); } + + public MVStoreConfig clone() { + MVStoreConfig config = new MVStoreConfig(); + config.eventListeners(new HashSet<>(eventListeners)); + config.filePath(filePath); + config.autoCommitBufferSize(autoCommitBufferSize); + config.encryptionKey(encryptionKey); + config.isReadOnly(isReadOnly); + config.compress(compress); + config.compressHigh(compressHigh); + config.autoCommit(autoCommit); + config.autoCompact(autoCompact); + config.recoveryMode(recoveryMode); + config.cacheSize(cacheSize); + config.cacheConcurrency(cacheConcurrency); + config.pageSplitSize(pageSplitSize); + config.fileStore(fileStore); + return config; + } } diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreUtils.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreUtils.java index 48f387817..bfaf8c280 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreUtils.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreUtils.java @@ -17,12 +17,13 @@ package org.dizitart.no2.mvstore; import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.NitriteIOException; -import org.dizitart.no2.mvstore.compat.v3.MigrationUtil; +import org.dizitart.no2.mvstore.compat.v1.UpgradeUtil; import org.h2.mvstore.MVMap; import org.h2.mvstore.MVStore; +import org.h2.mvstore.MVStoreException; import java.io.File; @@ -31,12 +32,13 @@ import static org.dizitart.no2.common.util.StringUtils.isNullOrEmpty; /** - * @since 4.0.0 * @author Anindya Chatterjee. + * @since 4.0.0 */ @Slf4j class MVStoreUtils { - private MVStoreUtils() { } + private MVStoreUtils() { + } static MVStore openOrCreate(MVStoreConfig storeConfig) { MVStore.Builder builder = createBuilder(storeConfig); @@ -46,49 +48,50 @@ static MVStore openOrCreate(MVStoreConfig storeConfig) { try { store = builder.open(); testForMigration(store); - } catch (IllegalStateException ise) { - if (ise.getMessage().contains("file is locked")) { - throw new NitriteIOException("database is already opened in other process"); + } catch (MVStoreException me) { + if (me.getMessage().contains("file is locked")) { + throw new NitriteIOException("Database is already opened in other process"); } if (dbFile != null) { try { if (dbFile.isDirectory()) { - throw new NitriteIOException(storeConfig.filePath() - + " is a directory, must be a file"); + throw new NitriteIOException(storeConfig.filePath() + " is a directory, must be a file"); } if (dbFile.exists() && dbFile.isFile()) { - if (isCompatibilityError(ise)) { + if (isCompatibilityError(me)) { if (store != null) { store.closeImmediately(); } - store = tryMigrate(dbFile, builder, storeConfig); + + // try upgrading the database + store = tryUpgrade(dbFile, storeConfig); } else { - log.error("Database corruption detected. Trying to repair", ise); + log.error("Database corruption detected. Trying to repair", me); Recovery.recover(storeConfig.filePath()); store = builder.open(); } } else { if (storeConfig.isReadOnly()) { - throw new NitriteIOException("cannot create readonly database", ise); + throw new NitriteIOException("Cannot create readonly database", me); } } } catch (InvalidOperationException | NitriteIOException ex) { throw ex; } catch (Exception e) { - throw new NitriteIOException("database file is corrupted", e); + throw new NitriteIOException("Database file is corrupted", e); } } else { - throw new NitriteIOException("unable to create in-memory database", ise); + throw new NitriteIOException("Unable to create in-memory database", me); } } catch (IllegalArgumentException iae) { if (dbFile != null) { if (!dbFile.getParentFile().exists()) { - throw new NitriteIOException("directory " + dbFile.getParent() + " does not exists", iae); + throw new NitriteIOException("Directory " + dbFile.getParent() + " does not exists", iae); } } - throw new NitriteIOException("unable to create database file", iae); + throw new NitriteIOException("Unable to create database file", iae); } finally { if (store != null) { store.setRetentionTime(0); @@ -100,10 +103,8 @@ static MVStore openOrCreate(MVStoreConfig storeConfig) { return store; } - private static boolean isCompatibilityError(IllegalStateException ise) { - return ise.getCause() != null - && ise.getCause().getCause() instanceof ClassNotFoundException - && ise.getCause().getCause().getMessage().contains("org.dizitart.no2"); + private static boolean isCompatibilityError(Exception e) { + return e.getMessage().contains("The write format 1 is smaller than the supported format"); } private static MVStore.Builder createBuilder(MVStoreConfig mvStoreConfig) { @@ -130,7 +131,7 @@ private static MVStore.Builder createBuilder(MVStoreConfig mvStoreConfig) { if (mvStoreConfig.isReadOnly()) { if (isNullOrEmpty(mvStoreConfig.filePath())) { - throw new InvalidOperationException("unable create readonly in-memory database"); + throw new InvalidOperationException("Unable create readonly in-memory database"); } builder = builder.readOnly(); } @@ -167,24 +168,17 @@ private static MVStore.Builder createBuilder(MVStoreConfig mvStoreConfig) { return builder; } - private static MVStore tryMigrate(File orgFile, MVStore.Builder builder, MVStoreConfig storeConfig) { - log.info("Migrating old database format to new database format"); - - // open old store with builder - MVStore oldMvStore = builder.open(); - + private static MVStore tryUpgrade(File orgFile, MVStoreConfig storeConfig) { // create new store with builder File newFile = new File(orgFile.getPath() + "_new"); - storeConfig.filePath(newFile.getPath()); - MVStore.Builder newBuilder = createBuilder(storeConfig); - MVStore newMvStore = newBuilder.open(); + MVStoreConfig newStoreConfig = storeConfig.clone(); + newStoreConfig.filePath(newFile.getPath()); + MVStore.Builder newBuilder = createBuilder(newStoreConfig); - // migrate 2 stores maps - MigrationUtil.migrate(newMvStore, oldMvStore); - switchFiles(newFile, orgFile); + UpgradeUtil.tryUpgrade(newBuilder, storeConfig); - // open new store calling openOrCreate and return - storeConfig.filePath(orgFile.getPath()); + // switch the file + switchFiles(newFile, orgFile); return openOrCreate(storeConfig); } @@ -192,14 +186,14 @@ private static void switchFiles(File newFile, File orgFile) { File backupFile = new File(orgFile.getPath() + "_old"); if (orgFile.renameTo(backupFile)) { if (!newFile.renameTo(orgFile)) { - throw new NitriteIOException("could not rename new data file"); + throw new NitriteIOException("Could not rename new data file"); } if (!backupFile.delete()) { - throw new NitriteIOException("could not delete backup data file"); + throw new NitriteIOException("Could not delete backup data file"); } } else { - throw new NitriteIOException("could not create backup copy of old data file"); + throw new NitriteIOException("Could not create backup copy of old data file"); } } @@ -212,6 +206,8 @@ private static void testForMigration(MVStore store) { MVStore.TxCounter txCounter = store.registerVersionUsage(); MVMap metaMap = store.openMap(META_MAP_NAME); try { + // fire one operation to trigger compatibility issue + // if no exception thrown, then the database is compatible metaMap.remove("MigrationTest"); } catch (IllegalStateException e) { store.close(); diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVMap.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVMap.java index 08703bccc..23901cd2c 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVMap.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVMap.java @@ -195,6 +195,11 @@ public void drop() { } } + @Override + public boolean isDropped() { + return droppedFlag.get(); + } + @Override public void close() { if (!closedFlag.get() && !droppedFlag.get()) { @@ -202,4 +207,9 @@ public void close() { nitriteStore.closeMap(getName()); } } + + @Override + public boolean isClosed() { + return closedFlag.get(); + } } diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVRTreeMap.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVRTreeMap.java index b8c373abb..aa0a111ea 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVRTreeMap.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVRTreeMap.java @@ -23,7 +23,6 @@ import org.dizitart.no2.store.NitriteStore; import org.h2.mvstore.MVStore; import org.h2.mvstore.rtree.MVRTreeMap; -import org.h2.mvstore.rtree.SpatialKey; import java.util.Iterator; @@ -45,7 +44,7 @@ class NitriteMVRTreeMap implements NitriteRTree< @Override public void add(Key key, NitriteId nitriteId) { if (nitriteId != null && nitriteId.getIdValue() != null) { - SpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue())); + MVSpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue())); MVStore.TxCounter txCounter = mvStore.registerVersionUsage(); try { mvMap.add(spatialKey, key); @@ -58,7 +57,7 @@ public void add(Key key, NitriteId nitriteId) { @Override public void remove(Key key, NitriteId nitriteId) { if (nitriteId != null && nitriteId.getIdValue() != null) { - SpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue())); + MVSpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue())); MVStore.TxCounter txCounter = mvStore.registerVersionUsage(); try { mvMap.remove(spatialKey); @@ -70,15 +69,15 @@ public void remove(Key key, NitriteId nitriteId) { @Override public RecordStream findIntersectingKeys(Key key) { - SpatialKey spatialKey = getKey(key, 0L); - MVRTreeMap.RTreeCursor treeCursor = mvMap.findIntersectingKeys(spatialKey); + MVSpatialKey spatialKey = getKey(key, 0L); + MVRTreeMap.RTreeCursor treeCursor = mvMap.findIntersectingKeys(spatialKey); return getRecordStream(treeCursor); } @Override public RecordStream findContainedKeys(Key key) { - SpatialKey spatialKey = getKey(key, 0L); - MVRTreeMap.RTreeCursor treeCursor = mvMap.findContainedKeys(spatialKey); + MVSpatialKey spatialKey = getKey(key, 0L); + MVRTreeMap.RTreeCursor treeCursor = mvMap.findContainedKeys(spatialKey); return getRecordStream(treeCursor); } @@ -87,16 +86,16 @@ public long size() { return mvMap.sizeAsLong(); } - private SpatialKey getKey(Key key, long id) { + private MVSpatialKey getKey(Key key, long id) { if (key == null) { - return new SpatialKey(id); + return new MVSpatialKey(id); } else { - return new SpatialKey(id, key.getMinX(), + return new MVSpatialKey(id, key.getMinX(), key.getMaxX(), key.getMinY(), key.getMaxY()); } } - private RecordStream getRecordStream(MVRTreeMap.RTreeCursor treeCursor) { + private RecordStream getRecordStream(MVRTreeMap.RTreeCursor treeCursor) { return RecordStream.fromIterable(() -> new Iterator() { @Override public boolean hasNext() { @@ -105,7 +104,7 @@ public boolean hasNext() { @Override public NitriteId next() { - SpatialKey next = treeCursor.next(); + MVSpatialKey next = (MVSpatialKey) treeCursor.next(); return NitriteId.createId(Long.toString(next.getId())); } }); diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/Recovery.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/Recovery.java index 358184810..ce9b79b23 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/Recovery.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/Recovery.java @@ -248,11 +248,11 @@ private static Chunk readChunkHeader(ByteBuffer buff, long start) { } } catch (Exception e) { // there could be various reasons - throw DataUtils.newIllegalStateException( + throw DataUtils.newMVStoreException( DataUtils.ERROR_FILE_CORRUPT, "File corrupt reading chunk at position {0}", start, e); } - throw DataUtils.newIllegalStateException( + throw DataUtils.newMVStoreException( DataUtils.ERROR_FILE_CORRUPT, "File corrupt reading chunk at position {0}", start); } diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/Compat.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/Compat.java similarity index 98% rename from nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/Compat.java rename to nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/Compat.java index a0b2d8393..9fee00913 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/Compat.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/Compat.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.dizitart.no2.mvstore.compat.v3; +package org.dizitart.no2.mvstore.compat.v1; import lombok.Data; diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/MVMapBuilder.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/MVMapBuilder.java similarity index 80% rename from nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/MVMapBuilder.java rename to nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/MVMapBuilder.java index b83a8c8b6..8f376e7d5 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/MVMapBuilder.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/MVMapBuilder.java @@ -14,9 +14,10 @@ * limitations under the License. */ -package org.dizitart.no2.mvstore.compat.v3; +package org.dizitart.no2.mvstore.compat.v1; -import org.h2.mvstore.MVMap; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.MVMap; /** * The type Mv map builder. @@ -32,7 +33,7 @@ class MVMapBuilder extends MVMap.Builder { * Instantiates a new Mv map builder. */ public MVMapBuilder() { - setKeyType(new org.dizitart.no2.mvstore.compat.v3.NitriteDataType()); - setValueType(new org.dizitart.no2.mvstore.compat.v3.NitriteDataType()); + setKeyType(new NitriteDataType()); + setValueType(new NitriteDataType()); } } diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/NitriteDataType.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/NitriteDataType.java similarity index 99% rename from nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/NitriteDataType.java rename to nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/NitriteDataType.java index 4ec5b385c..b4932cd05 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/NitriteDataType.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/NitriteDataType.java @@ -14,13 +14,14 @@ * limitations under the License. */ -package org.dizitart.no2.mvstore.compat.v3; +package org.dizitart.no2.mvstore.compat.v1; -import org.h2.mvstore.DataUtils; -import org.h2.mvstore.WriteBuffer; -import org.h2.mvstore.type.DataType; -import org.h2.mvstore.type.ObjectDataType; -import org.h2.mvstore.type.StringDataType; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.DataUtils; +import org.dizitart.no2.mvstore.compat.v1.mvstore.WriteBuffer; +import org.dizitart.no2.mvstore.compat.v1.mvstore.type.DataType; +import org.dizitart.no2.mvstore.compat.v1.mvstore.type.ObjectDataType; +import org.dizitart.no2.mvstore.compat.v1.mvstore.type.StringDataType; import org.h2.util.Utils; import java.io.ByteArrayInputStream; diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/NitriteObjectInputStream.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/NitriteObjectInputStream.java similarity index 98% rename from nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/NitriteObjectInputStream.java rename to nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/NitriteObjectInputStream.java index fc20a7793..d04a328f3 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/NitriteObjectInputStream.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/NitriteObjectInputStream.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.dizitart.no2.mvstore.compat.v3; +package org.dizitart.no2.mvstore.compat.v1; import lombok.extern.slf4j.Slf4j; diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/MigrationUtil.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/UpgradeUtil.java similarity index 73% rename from nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/MigrationUtil.java rename to nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/UpgradeUtil.java index 6c7119a21..3d3db1743 100644 --- a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v3/MigrationUtil.java +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/UpgradeUtil.java @@ -1,34 +1,38 @@ /* - * Copyright (c) 2019-2020. Nitrite author or authors. + * Copyright (c) 2017-2022 Nitrite author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ -package org.dizitart.no2.mvstore.compat.v3; +package org.dizitart.no2.mvstore.compat.v1; +import lombok.extern.slf4j.Slf4j; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.collection.meta.Attributes; import org.dizitart.no2.common.DBNull; +import org.dizitart.no2.common.DBValue; import org.dizitart.no2.common.Fields; +import org.dizitart.no2.common.meta.Attributes; +import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.ValidationException; -import org.dizitart.no2.index.DBValue; import org.dizitart.no2.index.IndexDescriptor; import org.dizitart.no2.index.IndexMeta; +import org.dizitart.no2.mvstore.MVStoreConfig; +import org.dizitart.no2.mvstore.compat.v1.mvstore.MVMap; +import org.dizitart.no2.mvstore.compat.v1.mvstore.MVStore; import org.dizitart.no2.store.UserCredential; -import org.h2.mvstore.MVMap; -import org.h2.mvstore.MVStore; import java.util.*; import java.util.concurrent.ConcurrentSkipListSet; @@ -37,45 +41,47 @@ import static org.dizitart.no2.common.Constants.INDEX_PREFIX; import static org.dizitart.no2.common.Constants.STORE_INFO; import static org.dizitart.no2.common.util.ObjectUtils.convertToObjectArray; +import static org.dizitart.no2.common.util.StringUtils.isNullOrEmpty; /** - * An utility class to migrate the. - * - * @since 4.0.0 * @author Anindya Chatterjee */ -public class MigrationUtil { - - /** - * Migrate an old 3.x compatible store to new 4.x compatible store. - * - * @param newStore the new store - * @param oldStore the old store - */ - @SuppressWarnings({"rawtypes"}) - public static void migrate(MVStore newStore, MVStore oldStore) { +@Slf4j +public class UpgradeUtil { + private UpgradeUtil() { + } + + public static void tryUpgrade(org.h2.mvstore.MVStore.Builder newBuilder, MVStoreConfig oldStoreConfig) { + log.info("Upgrading old database format to new database format"); + + MVStore.Builder oldBuilder = createBuilder(oldStoreConfig); + try (MVStore oldStore = oldBuilder.open()) { + try (org.h2.mvstore.MVStore newStore = newBuilder.open()) { + upgrade(newStore, oldStore); + } + } + } + + private static void upgrade(org.h2.mvstore.MVStore newStore, MVStore oldStore) { try { validateOldStore(oldStore); Set mapNames = oldStore.getMapNames(); for (String mapName : mapNames) { - MVMap oldMap = oldStore.openMap(mapName, new MVMapBuilder<>()); - MVMap newMap = newStore.openMap(mapName); + MVMap oldMap = oldStore.openMap(mapName, new MVMapBuilder<>()); + org.h2.mvstore.MVMap newMap = newStore.openMap(mapName); copyData(oldMap, newMap); } oldStore.commit(); newStore.commit(); } catch (Throwable t) { - throw new NitriteIOException("migration of old data has failed", t); - } finally { - oldStore.close(); - newStore.close(); + throw new NitriteIOException("Upgrade of old database has failed", t); } } @SuppressWarnings({"unchecked", "rawtypes"}) - private static void copyData(MVMap oldMap, MVMap newMap) { + private static void copyData(MVMap oldMap, org.h2.mvstore.MVMap newMap) { if (oldMap != null) { Set entrySet = oldMap.entrySet(); for (Map.Entry entry : entrySet) { @@ -167,7 +173,7 @@ private static Attributes attributes(Compat.Attributes value) { Attributes attributes = new Attributes(); attributes.set(Attributes.CREATED_TIME, Long.toString(value.getCreatedTime())); attributes.set(Attributes.LAST_MODIFIED_TIME, Long.toString(value.getLastModifiedTime())); - attributes.set(Attributes.LAST_SYNCED, Long.toString(value.getLastSynced())); + attributes.set(Attributes.LOCAL_COLLECTION_MARKER, Long.toString(value.getLastSynced())); attributes.set(Attributes.SYNC_LOCK, Long.toString(value.getSyncLock())); attributes.set(Attributes.EXPIRY_WAIT, Long.toString(value.getExpiryWait())); if (value.getCollection() != null) { @@ -220,7 +226,39 @@ private static UserCredential credential(Compat.UserCredential value) { private static void validateOldStore(MVStore store) { if (store.hasMap(STORE_INFO)) { - throw new ValidationException("database file is corrupted"); + throw new ValidationException("Database file is corrupted"); + } + } + + private static MVStore.Builder createBuilder(MVStoreConfig mvStoreConfig) { + MVStore.Builder builder = new MVStore.Builder(); + + // auto compact disabled github issue #41 + builder.autoCompactFillRate(0); + + if (!isNullOrEmpty(mvStoreConfig.filePath())) { + builder = builder.fileName(mvStoreConfig.filePath()); + } + + if (!mvStoreConfig.autoCommit()) { + builder = builder.autoCommitDisabled(); + } + + if (mvStoreConfig.autoCommitBufferSize() > 0) { + builder = builder.autoCommitBufferSize(mvStoreConfig.autoCommitBufferSize()); } + + if (mvStoreConfig.isReadOnly()) { + if (isNullOrEmpty(mvStoreConfig.filePath())) { + throw new InvalidOperationException("Unable create readonly in-memory database"); + } + builder = builder.readOnly(); + } + + if (mvStoreConfig.compress()) { + builder = builder.compress(); + } + + return builder; } } diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Chunk.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Chunk.java new file mode 100644 index 000000000..569c6f246 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Chunk.java @@ -0,0 +1,481 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Comparator; +import java.util.HashMap; + +/** + * A chunk of data, containing one or multiple pages. + *

+ * Chunks are page aligned (each page is usually 4096 bytes). + * There are at most 67 million (2^26) chunks, + * each chunk is at most 2 GB large. + */ +public class Chunk +{ + + /** + * The maximum chunk id. + */ + public static final int MAX_ID = (1 << 26) - 1; + + /** + * The maximum length of a chunk header, in bytes. + */ + static final int MAX_HEADER_LENGTH = 1024; + + /** + * The length of the chunk footer. The longest footer is: + * chunk:ffffffff,block:ffffffffffffffff, + * version:ffffffffffffffff,fletcher:ffffffff + */ + static final int FOOTER_LENGTH = 128; + + private static final String ATTR_CHUNK = "chunk"; + private static final String ATTR_BLOCK = "block"; + private static final String ATTR_LEN = "len"; + private static final String ATTR_MAP = "map"; + private static final String ATTR_MAX = "max"; + private static final String ATTR_NEXT = "next"; + private static final String ATTR_PAGES = "pages"; + private static final String ATTR_ROOT = "root"; + private static final String ATTR_TIME = "time"; + private static final String ATTR_VERSION = "version"; + private static final String ATTR_LIVE_MAX = "liveMax"; + private static final String ATTR_LIVE_PAGES = "livePages"; + private static final String ATTR_UNUSED = "unused"; + private static final String ATTR_UNUSED_AT_VERSION = "unusedAtVersion"; + private static final String ATTR_PIN_COUNT = "pinCount"; + private static final String ATTR_FLETCHER = "fletcher"; + + /** + * The chunk id. + */ + public final int id; + + /** + * The start block number within the file. + */ + public volatile long block; + + /** + * The length in number of blocks. + */ + public int len; + + /** + * The total number of pages in this chunk. + */ + int pageCount; + + /** + * The number of pages still alive. + */ + int pageCountLive; + + /** + * The sum of the max length of all pages. + */ + public long maxLen; + + /** + * The sum of the max length of all pages that are in use. + */ + public long maxLenLive; + + /** + * The garbage collection priority. Priority 0 means it needs to be + * collected, a high value means low priority. + */ + int collectPriority; + + /** + * The position of the meta root. + */ + long metaRootPos; + + /** + * The version stored in this chunk. + */ + public long version; + + /** + * When this chunk was created, in milliseconds after the store was created. + */ + public long time; + + /** + * When this chunk was no longer needed, in milliseconds after the store was + * created. After this, the chunk is kept alive a bit longer (in case it is + * referenced in older versions). + */ + public long unused; + + /** + * Version of the store at which chunk become unused and therefore can be + * considered "dead" and collected after this version is no longer in use. + */ + long unusedAtVersion; + + /** + * The last used map id. + */ + public int mapId; + + /** + * The predicted position of the next chunk. + */ + public long next; + + /** + * Number of live pinned pages. + */ + private int pinCount; + + + Chunk(int id) { + this.id = id; + } + + /** + * Read the header from the byte buffer. + * + * @param buff the source buffer + * @param start the start of the chunk in the file + * @return the chunk + */ + static Chunk readChunkHeader(ByteBuffer buff, long start) { + int pos = buff.position(); + byte[] data = new byte[Math.min(buff.remaining(), MAX_HEADER_LENGTH)]; + buff.get(data); + try { + for (int i = 0; i < data.length; i++) { + if (data[i] == '\n') { + // set the position to the start of the first page + buff.position(pos + i + 1); + String s = new String(data, 0, i, StandardCharsets.ISO_8859_1).trim(); + return fromString(s); + } + } + } catch (Exception e) { + // there could be various reasons + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_CORRUPT, + "File corrupt reading chunk at position {0}", start, e); + } + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_CORRUPT, + "File corrupt reading chunk at position {0}", start); + } + + /** + * Write the chunk header. + * + * @param buff the target buffer + * @param minLength the minimum length + */ + void writeChunkHeader(WriteBuffer buff, int minLength) { + long delimiterPosition = buff.position() + minLength - 1; + buff.put(asString().getBytes(StandardCharsets.ISO_8859_1)); + while (buff.position() < delimiterPosition) { + buff.put((byte) ' '); + } + if (minLength != 0 && buff.position() > delimiterPosition) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_INTERNAL, + "Chunk metadata too long"); + } + buff.put((byte) '\n'); + } + + /** + * Get the metadata key for the given chunk id. + * + * @param chunkId the chunk id + * @return the metadata key + */ + static String getMetaKey(int chunkId) { + return ATTR_CHUNK + "." + Integer.toHexString(chunkId); + } + + /** + * Build a block from the given string. + * + * @param s the string + * @return the block + */ + public static Chunk fromString(String s) { + HashMap map = DataUtils.parseMap(s); + int id = DataUtils.readHexInt(map, ATTR_CHUNK, 0); + Chunk c = new Chunk(id); + c.block = DataUtils.readHexLong(map, ATTR_BLOCK, 0); + c.len = DataUtils.readHexInt(map, ATTR_LEN, 0); + c.pageCount = DataUtils.readHexInt(map, ATTR_PAGES, 0); + c.pageCountLive = DataUtils.readHexInt(map, ATTR_LIVE_PAGES, c.pageCount); + c.mapId = DataUtils.readHexInt(map, ATTR_MAP, 0); + c.maxLen = DataUtils.readHexLong(map, ATTR_MAX, 0); + c.maxLenLive = DataUtils.readHexLong(map, ATTR_LIVE_MAX, c.maxLen); + c.metaRootPos = DataUtils.readHexLong(map, ATTR_ROOT, 0); + c.time = DataUtils.readHexLong(map, ATTR_TIME, 0); + c.unused = DataUtils.readHexLong(map, ATTR_UNUSED, 0); + c.unusedAtVersion = DataUtils.readHexLong(map, ATTR_UNUSED_AT_VERSION, 0); + c.version = DataUtils.readHexLong(map, ATTR_VERSION, id); + c.next = DataUtils.readHexLong(map, ATTR_NEXT, 0); + c.pinCount = DataUtils.readHexInt(map, ATTR_PIN_COUNT, 0); + return c; + } + + /** + * Calculate the fill rate in %. 0 means empty, 100 means full. + * + * @return the fill rate + */ + int getFillRate() { + assert maxLenLive <= maxLen : maxLenLive + " > " + maxLen; + if (maxLenLive <= 0) { + return 0; + } else if (maxLenLive == maxLen) { + return 100; + } + return 1 + (int) (98 * maxLenLive / maxLen); + } + + @Override + public int hashCode() { + return id; + } + + @Override + public boolean equals(Object o) { + return o instanceof Chunk && ((Chunk) o).id == id; + } + + /** + * Get the chunk data as a string. + * + * @return the string + */ + public String asString() { + StringBuilder buff = new StringBuilder(240); + DataUtils.appendMap(buff, ATTR_CHUNK, id); + DataUtils.appendMap(buff, ATTR_BLOCK, block); + DataUtils.appendMap(buff, ATTR_LEN, len); + if (maxLen != maxLenLive) { + DataUtils.appendMap(buff, ATTR_LIVE_MAX, maxLenLive); + } + if (pageCount != pageCountLive) { + DataUtils.appendMap(buff, ATTR_LIVE_PAGES, pageCountLive); + } + DataUtils.appendMap(buff, ATTR_MAP, mapId); + DataUtils.appendMap(buff, ATTR_MAX, maxLen); + if (next != 0) { + DataUtils.appendMap(buff, ATTR_NEXT, next); + } + DataUtils.appendMap(buff, ATTR_PAGES, pageCount); + DataUtils.appendMap(buff, ATTR_ROOT, metaRootPos); + DataUtils.appendMap(buff, ATTR_TIME, time); + if (unused != 0) { + DataUtils.appendMap(buff, ATTR_UNUSED, unused); + } + if (unusedAtVersion != 0) { + DataUtils.appendMap(buff, ATTR_UNUSED_AT_VERSION, unusedAtVersion); + } + DataUtils.appendMap(buff, ATTR_VERSION, version); + DataUtils.appendMap(buff, ATTR_PIN_COUNT, pinCount); + return buff.toString(); + } + + byte[] getFooterBytes() { + StringBuilder buff = new StringBuilder(FOOTER_LENGTH); + DataUtils.appendMap(buff, ATTR_CHUNK, id); + DataUtils.appendMap(buff, ATTR_BLOCK, block); + DataUtils.appendMap(buff, ATTR_VERSION, version); + byte[] bytes = buff.toString().getBytes(StandardCharsets.ISO_8859_1); + int checksum = DataUtils.getFletcher32(bytes, 0, bytes.length); + DataUtils.appendMap(buff, ATTR_FLETCHER, checksum); + while (buff.length() < FOOTER_LENGTH - 1) { + buff.append(' '); + } + buff.append('\n'); + return buff.toString().getBytes(StandardCharsets.ISO_8859_1); + } + + boolean isSaved() { + return block != Long.MAX_VALUE; + } + + boolean isLive() { + return pageCountLive > 0; + } + + boolean isRewritable() { + return isSaved() + && isLive() + && pageCountLive < pageCount // not fully occupied + && isEvacuatable(); + } + + private boolean isEvacuatable() { + return pinCount == 0; + } + + /** + * Read a page of data into a ByteBuffer. + * + * @param fileStore to use + * @param pos page pos + * @param expectedMapId expected map id for the page + * @return ByteBuffer containing page data. + */ + ByteBuffer readBufferForPage(FileStore fileStore, long pos, int expectedMapId) { + assert isSaved() : this; + while (true) { + long originalBlock = block; + try { + long filePos = originalBlock * MVStore.BLOCK_SIZE; + long maxPos = filePos + len * MVStore.BLOCK_SIZE; + filePos += DataUtils.getPageOffset(pos); + if (filePos < 0) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_CORRUPT, + "Negative position {0}; p={1}, c={2}", filePos, pos, toString()); + } + + int length = DataUtils.getPageMaxLength(pos); + if (length == DataUtils.PAGE_LARGE) { + // read the first bytes to figure out actual length + length = fileStore.readFully(filePos, 128).getInt(); + } + length = (int) Math.min(maxPos - filePos, length); + if (length < 0) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT, + "Illegal page length {0} reading at {1}; max pos {2} ", length, filePos, maxPos); + } + + ByteBuffer buff = fileStore.readFully(filePos, length); + + int offset = DataUtils.getPageOffset(pos); + int start = buff.position(); + int remaining = buff.remaining(); + int pageLength = buff.getInt(); + if (pageLength > remaining || pageLength < 4) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT, + "File corrupted in chunk {0}, expected page length 4..{1}, got {2}", id, remaining, + pageLength); + } + buff.limit(start + pageLength); + + short check = buff.getShort(); + int checkTest = DataUtils.getCheckValue(id) + ^ DataUtils.getCheckValue(offset) + ^ DataUtils.getCheckValue(pageLength); + if (check != (short) checkTest) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT, + "File corrupted in chunk {0}, expected check value {1}, got {2}", id, checkTest, check); + } + + int mapId = DataUtils.readVarInt(buff); + if (mapId != expectedMapId) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT, + "File corrupted in chunk {0}, expected map id {1}, got {2}", id, expectedMapId, mapId); + } + + if (originalBlock == block) { + return buff; + } + } catch (IllegalStateException ex) { + if (originalBlock == block) { + throw ex; + } + } + } + } + + /** + * Modifies internal state to reflect the fact that one more page is stored + * within this chunk. + * + * @param pageLengthOnDisk + * size of the page + * @param singleWriter + * indicates whether page belongs to append mode capable map + * (single writer map). Such pages are "pinned" to the chunk, + * they can't be evacuated (moved to a different chunk) while + * on-line, but they assumed to be short-lived anyway. + */ + void accountForWrittenPage(int pageLengthOnDisk, boolean singleWriter) { + maxLen += pageLengthOnDisk; + pageCount++; + maxLenLive += pageLengthOnDisk; + pageCountLive++; + if (singleWriter) { + pinCount++; + } + } + + /** + * Modifies internal state to reflect the fact that one the pages within + * this chunk was removed from the map. + * + * @param pageLength + * on disk of the removed page + * @param pinned + * whether removed page was pinned + * @param now + * is a moment in time (since creation of the store), when + * removal is recorded, and retention period starts + * @param version + * at which page was removed + * @return true if all of the pages, this chunk contains, were already + * removed, and false otherwise + */ + boolean accountForRemovedPage(int pageLength, boolean pinned, long now, long version) { + assert isSaved() : this; + maxLenLive -= pageLength; + pageCountLive--; + if (pinned) { + pinCount--; + } + + if (unusedAtVersion < version) { + unusedAtVersion = version; + } + + assert pinCount >= 0 : this; + assert pageCountLive >= 0 : this; + assert pinCount <= pageCountLive : this; + assert maxLenLive >= 0 : this; + assert (pageCountLive == 0) == (maxLenLive == 0) : this; + + if (!isLive()) { + assert isEvacuatable() : this; + unused = now; + return true; + } + return false; + } + + @Override + public String toString() { + return asString(); + } + + + public static final class PositionComparator implements Comparator { + public static final Comparator INSTANCE = new PositionComparator(); + + private PositionComparator() {} + + @Override + public int compare(Chunk one, Chunk two) { + return Long.compare(one.block, two.block); + } + } +} + diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Cursor.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Cursor.java new file mode 100644 index 000000000..80c96c8e0 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Cursor.java @@ -0,0 +1,163 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A cursor to iterate over elements in ascending order. + * + * @param the key type + * @param the value type + */ +public class Cursor implements Iterator { + private final K to; + private CursorPos cursorPos; + private CursorPos keeper; + private K current; + private K last; + private V lastValue; + private Page lastPage; + + public Cursor(Page root, K from) { + this(root, from, null); + } + + public Cursor(Page root, K from, K to) { + this.cursorPos = traverseDown(root, from); + this.to = to; + } + + @Override + @SuppressWarnings("unchecked") + public boolean hasNext() { + if (cursorPos != null) { + while (current == null) { + Page page = cursorPos.page; + int index = cursorPos.index; + if (index >= (page.isLeaf() ? page.getKeyCount() : page.map.getChildPageCount(page))) { + CursorPos tmp = cursorPos; + cursorPos = cursorPos.parent; + tmp.parent = keeper; + keeper = tmp; + if(cursorPos == null) + { + return false; + } + } else { + while (!page.isLeaf()) { + page = page.getChildPage(index); + if (keeper == null) { + cursorPos = new CursorPos(page, 0, cursorPos); + } else { + CursorPos tmp = keeper; + keeper = keeper.parent; + tmp.parent = cursorPos; + tmp.page = page; + tmp.index = 0; + cursorPos = tmp; + } + index = 0; + } + if (index < page.getKeyCount()) { + K key = (K) page.getKey(index); + if (to != null && page.map.getKeyType().compare(key, to) > 0) { + return false; + } + current = last = key; + lastValue = (V) page.getValue(index); + lastPage = page; + } + } + ++cursorPos.index; + } + } + return current != null; + } + + @Override + public K next() { + if(!hasNext()) { + throw new NoSuchElementException(); + } + current = null; + return last; + } + + /** + * Get the last read key if there was one. + * + * @return the key or null + */ + public K getKey() { + return last; + } + + /** + * Get the last read value if there was one. + * + * @return the value or null + */ + public V getValue() { + return lastValue; + } + + /** + * Get the page where last retrieved key is located. + * + * @return the page + */ + Page getPage() { + return lastPage; + } + + /** + * Skip over that many entries. This method is relatively fast (for this map + * implementation) even if many entries need to be skipped. + * + * @param n the number of entries to skip + */ + public void skip(long n) { + if (n < 10) { + while (n-- > 0 && hasNext()) { + next(); + } + } else if(hasNext()) { + assert cursorPos != null; + CursorPos cp = cursorPos; + CursorPos parent; + while ((parent = cp.parent) != null) cp = parent; + Page root = cp.page; + @SuppressWarnings("unchecked") + MVMap map = (MVMap) root.map; + long index = map.getKeyIndex(next()); + last = map.getKey(index + n); + this.cursorPos = traverseDown(root, last); + } + } + + @Override + public void remove() { + throw DataUtils.newUnsupportedOperationException( + "Removal is not supported"); + } + + /** + * Fetch the next entry that is equal or larger than the given key, starting + * from the given page. This method retains the stack. + * + * @param p the page to start from + * @param key the key to search, null means search for the first key + */ + private static CursorPos traverseDown(Page p, Object key) { + CursorPos cursorPos = key == null ? p.getPrependCursorPos(null) : CursorPos.traverseDown(p, key); + if (cursorPos.index < 0) { + cursorPos.index = -cursorPos.index - 1; + } + return cursorPos; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/CursorPos.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/CursorPos.java new file mode 100644 index 000000000..ca4bd35b8 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/CursorPos.java @@ -0,0 +1,86 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +/** + * A position in a cursor. + * Instance represents a node in the linked list, which traces path + * from a specific (target) key within a leaf node all the way up to te root + * (bottom up path). + */ +public class CursorPos +{ + /** + * The page at the current level. + */ + public Page page; + + /** + * Index of the key (within page above) used to go down to a lower level + * in case of intermediate nodes, or index of the target key for leaf a node. + * In a later case, it could be negative, if the key is not present. + */ + public int index; + + /** + * Next node in the linked list, representing the position within parent level, + * or null, if we are at the root level already. + */ + public CursorPos parent; + + + public CursorPos(Page page, int index, CursorPos parent) { + this.page = page; + this.index = index; + this.parent = parent; + } + + /** + * Searches for a given key and creates a breadcrumb trail through a B-tree + * rooted at a given Page. Resulting path starts at "insertion point" for a + * given key and goes back to the root. + * + * @param page root of the tree + * @param key the key to search for + * @return head of the CursorPos chain (insertion point) + */ + public static CursorPos traverseDown(Page page, Object key) { + CursorPos cursorPos = null; + while (!page.isLeaf()) { + int index = page.binarySearch(key) + 1; + if (index < 0) { + index = -index; + } + cursorPos = new CursorPos(page, index, cursorPos); + page = page.getChildPage(index); + } + return new CursorPos(page, page.binarySearch(key), cursorPos); + } + + /** + * Calculate the memory used by changes that are not yet stored. + * + * @param version the version + * @return the amount of memory + */ + int processRemovalInfo(long version) { + int unsavedMemory = 0; + for (CursorPos head = this; head != null; head = head.parent) { + unsavedMemory += head.page.removePage(version); + } + return unsavedMemory; + } + + @Override + public String toString() { + return "CursorPos{" + + "page=" + page + + ", index=" + index + + ", parent=" + parent + + '}'; + } +} + diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/DataUtils.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/DataUtils.java new file mode 100644 index 000000000..2d41e6fd3 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/DataUtils.java @@ -0,0 +1,1125 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.util.StringUtils; +import org.h2.engine.Constants; + +import java.io.EOFException; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * Utility methods + */ +public final class DataUtils { + + /** + * An error occurred while reading from the file. + */ + public static final int ERROR_READING_FAILED = 1; + + /** + * An error occurred when trying to write to the file. + */ + public static final int ERROR_WRITING_FAILED = 2; + + /** + * An internal error occurred. This could be a bug, or a memory corruption + * (for example caused by out of memory). + */ + public static final int ERROR_INTERNAL = 3; + + /** + * The object is already closed. + */ + public static final int ERROR_CLOSED = 4; + + /** + * The file format is not supported. + */ + public static final int ERROR_UNSUPPORTED_FORMAT = 5; + + /** + * The file is corrupt or (for encrypted files) the encryption key is wrong. + */ + public static final int ERROR_FILE_CORRUPT = 6; + + /** + * The file is locked. + */ + public static final int ERROR_FILE_LOCKED = 7; + + /** + * An error occurred when serializing or de-serializing. + */ + public static final int ERROR_SERIALIZATION = 8; + + /** + * The application was trying to read data from a chunk that is no longer + * available. + */ + public static final int ERROR_CHUNK_NOT_FOUND = 9; + + /** + * The block in the stream store was not found. + */ + public static final int ERROR_BLOCK_NOT_FOUND = 50; + + /** + * The transaction store is corrupt. + */ + public static final int ERROR_TRANSACTION_CORRUPT = 100; + + /** + * An entry is still locked by another transaction. + */ + public static final int ERROR_TRANSACTION_LOCKED = 101; + + /** + * There are too many open transactions. + */ + public static final int ERROR_TOO_MANY_OPEN_TRANSACTIONS = 102; + + /** + * The transaction store is in an illegal state (for example, not yet + * initialized). + */ + public static final int ERROR_TRANSACTION_ILLEGAL_STATE = 103; + + /** + * The transaction contains too many changes. + */ + public static final int ERROR_TRANSACTION_TOO_BIG = 104; + + /** + * Deadlock discovered and one of transactions involved chosen as victim and rolled back. + */ + public static final int ERROR_TRANSACTIONS_DEADLOCK = 105; + + /** + * The type for leaf page. + */ + public static final int PAGE_TYPE_LEAF = 0; + + /** + * The type for node page. + */ + public static final int PAGE_TYPE_NODE = 1; + + /** + * The bit mask for compressed pages (compression level fast). + */ + public static final int PAGE_COMPRESSED = 2; + + /** + * The bit mask for compressed pages (compression level high). + */ + public static final int PAGE_COMPRESSED_HIGH = 2 + 4; + + /** + * The maximum length of a variable size int. + */ + public static final int MAX_VAR_INT_LEN = 5; + + /** + * The maximum length of a variable size long. + */ + public static final int MAX_VAR_LONG_LEN = 10; + + /** + * The maximum integer that needs less space when using variable size + * encoding (only 3 bytes instead of 4). + */ + public static final int COMPRESSED_VAR_INT_MAX = 0x1fffff; + + /** + * The maximum long that needs less space when using variable size + * encoding (only 7 bytes instead of 8). + */ + public static final long COMPRESSED_VAR_LONG_MAX = 0x1ffffffffffffL; + + /** + * The marker size of a very large page. + */ + public static final int PAGE_LARGE = 2 * 1024 * 1024; + + // The following are key prefixes used in meta map + + /** + * The prefix for chunks ("chunk."). This, plus the chunk id (hex encoded) + * is the key, and the serialized chunk metadata is the value. + */ + public static final String META_CHUNK = "chunk."; + + /** + * The prefix for names ("name."). This, plus the name of the map, is the + * key, and the map id (hey encoded) is the value. + */ + public static final String META_NAME = "name."; + + /** + * The prefix for maps ("map."). This, plus the map id (hex encoded) is the + * key, and the serialized in the map metadata is the value. + */ + public static final String META_MAP = "map."; + + /** + * The prefix for root positions of maps ("root."). This, plus the map id + * (hex encoded) is the key, and the position (hex encoded) is the value. + */ + public static final String META_ROOT = "root."; + + /** + * Get the length of the variable size int. + * + * @param x the value + * @return the length in bytes + */ + public static int getVarIntLen(int x) { + if ((x & (-1 << 7)) == 0) { + return 1; + } else if ((x & (-1 << 14)) == 0) { + return 2; + } else if ((x & (-1 << 21)) == 0) { + return 3; + } else if ((x & (-1 << 28)) == 0) { + return 4; + } + return 5; + } + + /** + * Get the length of the variable size long. + * + * @param x the value + * @return the length in bytes + */ + public static int getVarLongLen(long x) { + int i = 1; + while (true) { + x >>>= 7; + if (x == 0) { + return i; + } + i++; + } + } + + /** + * Read a variable size int. + * + * @param buff the source buffer + * @return the value + */ + public static int readVarInt(ByteBuffer buff) { + int b = buff.get(); + if (b >= 0) { + return b; + } + // a separate function so that this one can be inlined + return readVarIntRest(buff, b); + } + + private static int readVarIntRest(ByteBuffer buff, int b) { + int x = b & 0x7f; + b = buff.get(); + if (b >= 0) { + return x | (b << 7); + } + x |= (b & 0x7f) << 7; + b = buff.get(); + if (b >= 0) { + return x | (b << 14); + } + x |= (b & 0x7f) << 14; + b = buff.get(); + if (b >= 0) { + return x | b << 21; + } + x |= ((b & 0x7f) << 21) | (buff.get() << 28); + return x; + } + + /** + * Read a variable size long. + * + * @param buff the source buffer + * @return the value + */ + public static long readVarLong(ByteBuffer buff) { + long x = buff.get(); + if (x >= 0) { + return x; + } + x &= 0x7f; + for (int s = 7; s < 64; s += 7) { + long b = buff.get(); + x |= (b & 0x7f) << s; + if (b >= 0) { + break; + } + } + return x; + } + + /** + * Write a variable size int. + * + * @param out the output stream + * @param x the value + * @throws IOException if some data could not be written + */ + public static void writeVarInt(OutputStream out, int x) throws IOException { + while ((x & ~0x7f) != 0) { + out.write((byte) (x | 0x80)); + x >>>= 7; + } + out.write((byte) x); + } + + /** + * Write a variable size int. + * + * @param buff the source buffer + * @param x the value + */ + public static void writeVarInt(ByteBuffer buff, int x) { + while ((x & ~0x7f) != 0) { + buff.put((byte) (x | 0x80)); + x >>>= 7; + } + buff.put((byte) x); + } + + /** + * Write characters from a string (without the length). + * + * @param buff the target buffer (must be large enough) + * @param s the string + * @param len the number of characters + */ + public static void writeStringData(ByteBuffer buff, + String s, int len) { + for (int i = 0; i < len; i++) { + int c = s.charAt(i); + if (c < 0x80) { + buff.put((byte) c); + } else if (c >= 0x800) { + buff.put((byte) (0xe0 | (c >> 12))); + buff.put((byte) (((c >> 6) & 0x3f))); + buff.put((byte) (c & 0x3f)); + } else { + buff.put((byte) (0xc0 | (c >> 6))); + buff.put((byte) (c & 0x3f)); + } + } + } + + /** + * Read a string. + * + * @param buff the source buffer + * @return the value + */ + public static String readString(ByteBuffer buff) { + return readString(buff, readVarInt(buff)); + } + + /** + * Read a string. + * + * @param buff the source buffer + * @param len the number of characters + * @return the value + */ + public static String readString(ByteBuffer buff, int len) { + char[] chars = new char[len]; + for (int i = 0; i < len; i++) { + int x = buff.get() & 0xff; + if (x < 0x80) { + chars[i] = (char) x; + } else if (x >= 0xe0) { + chars[i] = (char) (((x & 0xf) << 12) + + ((buff.get() & 0x3f) << 6) + (buff.get() & 0x3f)); + } else { + chars[i] = (char) (((x & 0x1f) << 6) + (buff.get() & 0x3f)); + } + } + return new String(chars); + } + + /** + * Write a variable size long. + * + * @param buff the target buffer + * @param x the value + */ + public static void writeVarLong(ByteBuffer buff, long x) { + while ((x & ~0x7f) != 0) { + buff.put((byte) (x | 0x80)); + x >>>= 7; + } + buff.put((byte) x); + } + + /** + * Write a variable size long. + * + * @param out the output stream + * @param x the value + * @throws IOException if some data could not be written + */ + public static void writeVarLong(OutputStream out, long x) + throws IOException { + while ((x & ~0x7f) != 0) { + out.write((byte) (x | 0x80)); + x >>>= 7; + } + out.write((byte) x); + } + + /** + * Copy the elements of an array, with a gap. + * + * @param src the source array + * @param dst the target array + * @param oldSize the size of the old array + * @param gapIndex the index of the gap + */ + public static void copyWithGap(Object src, Object dst, int oldSize, + int gapIndex) { + if (gapIndex > 0) { + System.arraycopy(src, 0, dst, 0, gapIndex); + } + if (gapIndex < oldSize) { + System.arraycopy(src, gapIndex, dst, gapIndex + 1, oldSize + - gapIndex); + } + } + + /** + * Copy the elements of an array, and remove one element. + * + * @param src the source array + * @param dst the target array + * @param oldSize the size of the old array + * @param removeIndex the index of the entry to remove + */ + public static void copyExcept(Object src, Object dst, int oldSize, + int removeIndex) { + if (removeIndex > 0 && oldSize > 0) { + System.arraycopy(src, 0, dst, 0, removeIndex); + } + if (removeIndex < oldSize) { + System.arraycopy(src, removeIndex + 1, dst, removeIndex, oldSize + - removeIndex - 1); + } + } + + /** + * Read from a file channel until the buffer is full. + * The buffer is rewind after reading. + * + * @param file the file channel + * @param pos the absolute position within the file + * @param dst the byte buffer + * @throws IllegalStateException if some data could not be read + */ + public static void readFully(FileChannel file, long pos, ByteBuffer dst) { + try { + do { + int len = file.read(dst, pos); + if (len < 0) { + throw new EOFException(); + } + pos += len; + } while (dst.remaining() > 0); + dst.rewind(); + } catch (IOException e) { + long size; + try { + size = file.size(); + } catch (IOException e2) { + size = -1; + } + throw newIllegalStateException( + ERROR_READING_FAILED, + "Reading from file {0} failed at {1} (length {2}), " + + "read {3}, remaining {4}", + file, pos, size, dst.position(), dst.remaining(), e); + } + } + + /** + * Write to a file channel. + * + * @param file the file channel + * @param pos the absolute position within the file + * @param src the source buffer + */ + public static void writeFully(FileChannel file, long pos, ByteBuffer src) { + try { + int off = 0; + do { + int len = file.write(src, pos + off); + off += len; + } while (src.remaining() > 0); + } catch (IOException e) { + throw newIllegalStateException( + ERROR_WRITING_FAILED, + "Writing to {0} failed; length {1} at {2}", + file, src.remaining(), pos, e); + } + } + + /** + * Convert the length to a length code 0..31. 31 means more than 1 MB. + * + * @param len the length + * @return the length code + */ + public static int encodeLength(int len) { + if (len <= 32) { + return 0; + } + int code = Integer.numberOfLeadingZeros(len); + int remaining = len << (code + 1); + code += code; + if ((remaining & (1 << 31)) != 0) { + code--; + } + if ((remaining << 1) != 0) { + code--; + } + code = Math.min(31, 52 - code); + // alternative code (slower): + // int x = len; + // int shift = 0; + // while (x > 3) { + // shift++; + // x = (x >>> 1) + (x & 1); + // } + // shift = Math.max(0, shift - 4); + // int code = (shift << 1) + (x & 1); + // code = Math.min(31, code); + return code; + } + + /** + * Get the chunk id from the position. + * + * @param pos the position + * @return the chunk id + */ + public static int getPageChunkId(long pos) { + return (int) (pos >>> 38); + } + + /** + * Get the maximum length for the given page position. + * + * @param pos the position + * @return the maximum length + */ + public static int getPageMaxLength(long pos) { + int code = (int) ((pos >> 1) & 31); + return decodePageLength(code); + } + + /** + * Get the maximum length for the given code. + * For the code 31, PAGE_LARGE is returned. + * + * @param code encoded page length + * @return the maximum length + */ + public static int decodePageLength(int code) { + if (code == 31) { + return PAGE_LARGE; + } + return (2 + (code & 1)) << ((code >> 1) + 4); + } + + /** + * Get the offset from the position. + * + * @param pos the position + * @return the offset + */ + public static int getPageOffset(long pos) { + return (int) (pos >> 6); + } + + /** + * Get the page type from the position. + * + * @param pos the position + * @return the page type (PAGE_TYPE_NODE or PAGE_TYPE_LEAF) + */ + public static int getPageType(long pos) { + return ((int) pos) & 1; + } + + /** + * Determines whether specified file position corresponds to a leaf page + * @param pos the position + * @return true if it is a leaf, false otherwise + */ + public static boolean isLeafPosition(long pos) { + return getPageType(pos) == PAGE_TYPE_LEAF; + } + + /** + * Find out if page was saved. + * + * @param pos the position + * @return true if page has been saved + */ + public static boolean isPageSaved(long pos) { + return (pos & ~1L) != 0; + } + + /** + * Find out if page was removed. + * + * @param pos the position + * @return true if page has been removed (no longer accessible from the + * current root of the tree) + */ + static boolean isPageRemoved(long pos) { + return pos == 1L; + } + + /** + * Get the position of this page. The following information is encoded in + * the position: the chunk id, the offset, the maximum length, and the type + * (node or leaf). + * + * @param chunkId the chunk id + * @param offset the offset + * @param length the length + * @param type the page type (1 for node, 0 for leaf) + * @return the position + */ + public static long getPagePos(int chunkId, int offset, + int length, int type) { + long pos = (long) chunkId << 38; + pos |= (long) offset << 6; + pos |= encodeLength(length) << 1; + pos |= type; + return pos; + } + + /** + * Calculate a check value for the given integer. A check value is mean to + * verify the data is consistent with a high probability, but not meant to + * protect against media failure or deliberate changes. + * + * @param x the value + * @return the check value + */ + public static short getCheckValue(int x) { + return (short) ((x >> 16) ^ x); + } + + /** + * Append a map to the string builder, sorted by key. + * + * @param buff the target buffer + * @param map the map + * @return the string builder + */ + public static StringBuilder appendMap(StringBuilder buff, HashMap map) { + Object[] keys = map.keySet().toArray(); + Arrays.sort(keys); + for (Object k : keys) { + String key = (String) k; + Object value = map.get(key); + if (value instanceof Long) { + appendMap(buff, key, (long) value); + } else if (value instanceof Integer) { + appendMap(buff, key, (int) value); + } else { + appendMap(buff, key, value.toString()); + } + } + return buff; + } + + private static StringBuilder appendMapKey(StringBuilder buff, String key) { + if (buff.length() > 0) { + buff.append(','); + } + return buff.append(key).append(':'); + } + + /** + * Append a key-value pair to the string builder. Keys may not contain a + * colon. Values that contain a comma or a double quote are enclosed in + * double quotes, with special characters escaped using a backslash. + * + * @param buff the target buffer + * @param key the key + * @param value the value + */ + public static void appendMap(StringBuilder buff, String key, String value) { + appendMapKey(buff, key); + if (value.indexOf(',') < 0 && value.indexOf('\"') < 0) { + buff.append(value); + } else { + buff.append('\"'); + for (int i = 0, size = value.length(); i < size; i++) { + char c = value.charAt(i); + if (c == '\"') { + buff.append('\\'); + } + buff.append(c); + } + buff.append('\"'); + } + } + + /** + * Append a key-value pair to the string builder. Keys may not contain a + * colon. + * + * @param buff the target buffer + * @param key the key + * @param value the value + */ + public static void appendMap(StringBuilder buff, String key, long value) { + appendMapKey(buff, key).append(Long.toHexString(value)); + } + + /** + * Append a key-value pair to the string builder. Keys may not contain a + * colon. + * + * @param buff the target buffer + * @param key the key + * @param value the value + */ + public static void appendMap(StringBuilder buff, String key, int value) { + appendMapKey(buff, key).append(Integer.toHexString(value)); + } + + /** + * @param buff output buffer, should be empty + * @param s parsed string + * @param i offset to parse from + * @param size stop offset (exclusive) + * @return new offset + */ + private static int parseMapValue(StringBuilder buff, String s, int i, int size) { + while (i < size) { + char c = s.charAt(i++); + if (c == ',') { + break; + } else if (c == '\"') { + while (i < size) { + c = s.charAt(i++); + if (c == '\\') { + if (i == size) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, "Not a map: {0}", s); + } + c = s.charAt(i++); + } else if (c == '\"') { + break; + } + buff.append(c); + } + } else { + buff.append(c); + } + } + return i; + } + + /** + * Parse a key-value pair list. + * + * @param s the list + * @return the map + * @throws IllegalStateException if parsing failed + */ + public static HashMap parseMap(String s) { + HashMap map = new HashMap<>(); + StringBuilder buff = new StringBuilder(); + for (int i = 0, size = s.length(); i < size;) { + int startKey = i; + i = s.indexOf(':', i); + if (i < 0) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, "Not a map: {0}", s); + } + String key = s.substring(startKey, i++); + i = parseMapValue(buff, s, i, size); + map.put(key, buff.toString()); + buff.setLength(0); + } + return map; + } + + /** + * Parse a key-value pair list and checks its checksum. + * + * @param bytes encoded map + * @return the map without mapping for {@code "fletcher"}, or {@code null} if checksum is wrong + * or parameter do not represent a properly formatted map serialization + */ + static HashMap parseChecksummedMap(byte[] bytes) { + int start = 0, end = bytes.length; + while (start < end && bytes[start] <= ' ') { + start++; + } + while (start < end && bytes[end - 1] <= ' ') { + end--; + } + String s = new String(bytes, start, end - start, StandardCharsets.ISO_8859_1); + HashMap map = new HashMap<>(); + StringBuilder buff = new StringBuilder(); + for (int i = 0, size = s.length(); i < size;) { + int startKey = i; + i = s.indexOf(':', i); + if (i < 0) { + // Corrupted map + return null; + } + if (i - startKey == 8 && s.regionMatches(startKey, "fletcher", 0, 8)) { + parseMapValue(buff, s, i + 1, size); + int check = (int) Long.parseLong(buff.toString(), 16); + if (check == getFletcher32(bytes, start, startKey - 1)) { + return map; + } + // Corrupted map + return null; + } + String key = s.substring(startKey, i++); + i = parseMapValue(buff, s, i, size); + map.put(key, buff.toString()); + buff.setLength(0); + } + // Corrupted map + return null; + } + + /** + * Parse a name from key-value pair list. + * + * @param s the list + * @return value of name item, or {@code null} + * @throws IllegalStateException if parsing failed + */ + public static String getMapName(String s) { + return getFromMap(s, "name"); + } + + /** + * Parse a specified pair from key-value pair list. + * + * @param s the list + * @param key the name of the key + * @return value of the specified item, or {@code null} + * @throws IllegalStateException if parsing failed + */ + public static String getFromMap(String s, String key) { + int keyLength = key.length(); + for (int i = 0, size = s.length(); i < size;) { + int startKey = i; + i = s.indexOf(':', i); + if (i < 0) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, "Not a map: {0}", s); + } + if (i++ - startKey == keyLength && s.regionMatches(startKey, key, 0, keyLength)) { + StringBuilder buff = new StringBuilder(); + parseMapValue(buff, s, i, size); + return buff.toString(); + } else { + while (i < size) { + char c = s.charAt(i++); + if (c == ',') { + break; + } else if (c == '\"') { + while (i < size) { + c = s.charAt(i++); + if (c == '\\') { + if (i++ == size) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, "Not a map: {0}", s); + } + } else if (c == '\"') { + break; + } + } + } + } + } + } + return null; + } + + /** + * Calculate the Fletcher32 checksum. + * + * @param bytes the bytes + * @param offset initial offset + * @param length the message length (if odd, 0 is appended) + * @return the checksum + */ + public static int getFletcher32(byte[] bytes, int offset, int length) { + int s1 = 0xffff, s2 = 0xffff; + int i = offset, len = offset + (length & ~1); + while (i < len) { + // reduce after 360 words (each word is two bytes) + for (int end = Math.min(i + 720, len); i < end;) { + int x = ((bytes[i++] & 0xff) << 8) | (bytes[i++] & 0xff); + s2 += s1 += x; + } + s1 = (s1 & 0xffff) + (s1 >>> 16); + s2 = (s2 & 0xffff) + (s2 >>> 16); + } + if ((length & 1) != 0) { + // odd length: append 0 + int x = (bytes[i] & 0xff) << 8; + s2 += s1 += x; + } + s1 = (s1 & 0xffff) + (s1 >>> 16); + s2 = (s2 & 0xffff) + (s2 >>> 16); + return (s2 << 16) | s1; + } + + /** + * Throw an IllegalArgumentException if the argument is invalid. + * + * @param test true if the argument is valid + * @param message the message + * @param arguments the arguments + * @throws IllegalArgumentException if the argument is invalid + */ + public static void checkArgument(boolean test, String message, + Object... arguments) { + if (!test) { + throw newIllegalArgumentException(message, arguments); + } + } + + /** + * Create a new IllegalArgumentException. + * + * @param message the message + * @param arguments the arguments + * @return the exception + */ + public static IllegalArgumentException newIllegalArgumentException( + String message, Object... arguments) { + return initCause(new IllegalArgumentException( + formatMessage(0, message, arguments)), + arguments); + } + + /** + * Create a new UnsupportedOperationException. + * + * @param message the message + * @return the exception + */ + public static UnsupportedOperationException + newUnsupportedOperationException(String message) { + return new UnsupportedOperationException(formatMessage(0, message)); + } + + /** + * Create a new IllegalStateException. + * + * @param errorCode the error code + * @param message the message + * @param arguments the arguments + * @return the exception + */ + public static IllegalStateException newIllegalStateException( + int errorCode, String message, Object... arguments) { + return initCause(new IllegalStateException( + formatMessage(errorCode, message, arguments)), + arguments); + } + + private static T initCause(T e, Object... arguments) { + int size = arguments.length; + if (size > 0) { + Object o = arguments[size - 1]; + if (o instanceof Throwable) { + e.initCause((Throwable) o); + } + } + return e; + } + + /** + * Format an error message. + * + * @param errorCode the error code + * @param message the message + * @param arguments the arguments + * @return the formatted message + */ + public static String formatMessage(int errorCode, String message, + Object... arguments) { + // convert arguments to strings, to avoid locale specific formatting + arguments = arguments.clone(); + for (int i = 0; i < arguments.length; i++) { + Object a = arguments[i]; + if (!(a instanceof Exception)) { + String s = a == null ? "null" : a.toString(); + if (s.length() > 1000) { + s = s.substring(0, 1000) + "..."; + } + arguments[i] = s; + } + } + return MessageFormat.format(message, arguments) + + " [" + Constants.VERSION_MAJOR + "." + + Constants.VERSION_MINOR + "." + Constants.BUILD_ID + + "/" + errorCode + "]"; + } + + /** + * Get the error code from an exception message. + * + * @param m the message + * @return the error code, or 0 if none + */ + public static int getErrorCode(String m) { + if (m != null && m.endsWith("]")) { + int dash = m.lastIndexOf('/'); + if (dash >= 0) { + try { + return StringUtils.parseUInt31(m, dash + 1, m.length() - 1); + } catch (NumberFormatException e) { + // no error code + } + } + } + return 0; + } + + /** + * Read a hex long value from a map. + * + * @param map the map + * @param key the key + * @param defaultValue if the value is null + * @return the parsed value + * @throws IllegalStateException if parsing fails + */ + public static long readHexLong(Map map, String key, long defaultValue) { + Object v = map.get(key); + if (v == null) { + return defaultValue; + } else if (v instanceof Long) { + return (Long) v; + } + try { + return parseHexLong((String) v); + } catch (NumberFormatException e) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, + "Error parsing the value {0}", v, e); + } + } + + /** + * Parse an unsigned, hex long. + * + * @param x the string + * @return the parsed value + * @throws IllegalStateException if parsing fails + */ + public static long parseHexLong(String x) { + try { + if (x.length() == 16) { + // avoid problems with overflow + // in Java 8, this special case is not needed + return (Long.parseLong(x.substring(0, 8), 16) << 32) | + Long.parseLong(x.substring(8, 16), 16); + } + return Long.parseLong(x, 16); + } catch (NumberFormatException e) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, + "Error parsing the value {0}", x, e); + } + } + + /** + * Parse an unsigned, hex long. + * + * @param x the string + * @return the parsed value + * @throws IllegalStateException if parsing fails + */ + public static int parseHexInt(String x) { + try { + // avoid problems with overflow + // in Java 8, we can use Integer.parseLong(x, 16); + return (int) Long.parseLong(x, 16); + } catch (NumberFormatException e) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, + "Error parsing the value {0}", x, e); + } + } + + /** + * Read a hex int value from a map. + * + * @param map the map + * @param key the key + * @param defaultValue if the value is null + * @return the parsed value + * @throws IllegalStateException if parsing fails + */ + public static int readHexInt(Map map, String key, int defaultValue) { + Object v = map.get(key); + if (v == null) { + return defaultValue; + } else if (v instanceof Integer) { + return (Integer) v; + } + try { + // support unsigned hex value + return (int) Long.parseLong((String) v, 16); + } catch (NumberFormatException e) { + throw newIllegalStateException(ERROR_FILE_CORRUPT, + "Error parsing the value {0}", v, e); + } + } + + /** + * Get the configuration parameter value, or default. + * + * @param config the configuration + * @param key the key + * @param defaultValue the default + * @return the configured value or default + */ + public static int getConfigParam(Map config, String key, int defaultValue) { + Object o = config.get(key); + if (o instanceof Number) { + return ((Number) o).intValue(); + } else if (o != null) { + try { + return Integer.decode(o.toString()); + } catch (NumberFormatException e) { + // ignore + } + } + return defaultValue; + } + +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/FileStore.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/FileStore.java new file mode 100644 index 000000000..28d552bb7 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/FileStore.java @@ -0,0 +1,437 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.cache.FilePathCache; +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePath; +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePathDisk; +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePathEncrypt; +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePathNio; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.util.concurrent.atomic.AtomicLong; + +/** + * The default storage mechanism of the MVStore. This implementation persists + * data to a file. The file store is responsible to persist data and for free + * space management. + */ +public class FileStore { + + /** + * The number of read operations. + */ + protected final AtomicLong readCount = new AtomicLong(); + + /** + * The number of read bytes. + */ + protected final AtomicLong readBytes = new AtomicLong(); + + /** + * The number of write operations. + */ + protected final AtomicLong writeCount = new AtomicLong(); + + /** + * The number of written bytes. + */ + protected final AtomicLong writeBytes = new AtomicLong(); + + /** + * The free spaces between the chunks. The first block to use is block 2 + * (the first two blocks are the store header). + */ + protected final FreeSpaceBitSet freeSpace = + new FreeSpaceBitSet(2, MVStore.BLOCK_SIZE); + + /** + * The file name. + */ + private String fileName; + + /** + * Whether this store is read-only. + */ + private boolean readOnly; + + /** + * The file size (cached). + */ + protected long fileSize; + + /** + * The file. + */ + private FileChannel file; + + /** + * The encrypted file (if encryption is used). + */ + private FileChannel encryptedFile; + + /** + * The file lock. + */ + private FileLock fileLock; + + @Override + public String toString() { + return fileName; + } + + /** + * Read from the file. + * + * @param pos the write position + * @param len the number of bytes to read + * @return the byte buffer + */ + public ByteBuffer readFully(long pos, int len) { + ByteBuffer dst = ByteBuffer.allocate(len); + DataUtils.readFully(file, pos, dst); + readCount.incrementAndGet(); + readBytes.addAndGet(len); + return dst; + } + + /** + * Write to the file. + * + * @param pos the write position + * @param src the source buffer + */ + public void writeFully(long pos, ByteBuffer src) { + int len = src.remaining(); + fileSize = Math.max(fileSize, pos + len); + DataUtils.writeFully(file, pos, src); + writeCount.incrementAndGet(); + writeBytes.addAndGet(len); + } + + /** + * Try to open the file. + * + * @param fileName the file name + * @param readOnly whether the file should only be opened in read-only mode, + * even if the file is writable + * @param encryptionKey the encryption key, or null if encryption is not + * used + */ + public void open(String fileName, boolean readOnly, char[] encryptionKey) { + if (file != null) { + return; + } + // ensure the Cache file system is registered + FilePathCache.INSTANCE.getScheme(); + FilePath p = FilePath.get(fileName); + // if no explicit scheme was specified, NIO is used + if (p instanceof FilePathDisk && + !fileName.startsWith(p.getScheme() + ":")) { + // ensure the NIO file system is registered + FilePathNio.class.getName(); + fileName = "nio:" + fileName; + } + this.fileName = fileName; + FilePath f = FilePath.get(fileName); + FilePath parent = f.getParent(); + if (parent != null && !parent.exists()) { + throw DataUtils.newIllegalArgumentException( + "Directory does not exist: {0}", parent); + } + if (f.exists() && !f.canWrite()) { + readOnly = true; + } + this.readOnly = readOnly; + try { + file = f.open(readOnly ? "r" : "rw"); + if (encryptionKey != null) { + byte[] key = FilePathEncrypt.getPasswordBytes(encryptionKey); + encryptedFile = file; + file = new FilePathEncrypt.FileEncrypt(fileName, key, file); + } + try { + if (readOnly) { + fileLock = file.tryLock(0, Long.MAX_VALUE, true); + } else { + fileLock = file.tryLock(); + } + } catch (OverlappingFileLockException e) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_LOCKED, + "The file is locked: {0}", fileName, e); + } + if (fileLock == null) { + try { close(); } catch (Exception ignore) {} + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_LOCKED, + "The file is locked: {0}", fileName); + } + fileSize = file.size(); + } catch (IOException e) { + try { close(); } catch (Exception ignore) {} + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_READING_FAILED, + "Could not open file {0}", fileName, e); + } + } + + /** + * Close this store. + */ + public void close() { + try { + if(file != null && file.isOpen()) { + if (fileLock != null) { + fileLock.release(); + } + file.close(); + } + } catch (Exception e) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_WRITING_FAILED, + "Closing failed for file {0}", fileName, e); + } finally { + fileLock = null; + file = null; + } + } + + /** + * Flush all changes. + */ + public void sync() { + if (file != null) { + try { + file.force(true); + } catch (IOException e) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_WRITING_FAILED, + "Could not sync file {0}", fileName, e); + } + } + } + + /** + * Get the file size. + * + * @return the file size + */ + public long size() { + return fileSize; + } + + /** + * Truncate the file. + * + * @param size the new file size + */ + public void truncate(long size) { + int attemptCount = 0; + while (true) { + try { + writeCount.incrementAndGet(); + file.truncate(size); + fileSize = Math.min(fileSize, size); + return; + } catch (IOException e) { + if (++attemptCount == 10) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_WRITING_FAILED, + "Could not truncate file {0} to size {1}", + fileName, size, e); + } + System.gc(); + Thread.yield(); + } + } + } + + /** + * Get the file instance in use. + *

+ * The application may read from the file (for example for online backup), + * but not write to it or truncate it. + * + * @return the file + */ + public FileChannel getFile() { + return file; + } + + /** + * Get the encrypted file instance, if encryption is used. + *

+ * The application may read from the file (for example for online backup), + * but not write to it or truncate it. + * + * @return the encrypted file, or null if encryption is not used + */ + public FileChannel getEncryptedFile() { + return encryptedFile; + } + + /** + * Get the number of write operations since this store was opened. + * For file based stores, this is the number of file write operations. + * + * @return the number of write operations + */ + public long getWriteCount() { + return writeCount.get(); + } + + /** + * Get the number of written bytes since this store was opened. + * + * @return the number of write operations + */ + public long getWriteBytes() { + return writeBytes.get(); + } + + /** + * Get the number of read operations since this store was opened. + * For file based stores, this is the number of file read operations. + * + * @return the number of read operations + */ + public long getReadCount() { + return readCount.get(); + } + + /** + * Get the number of read bytes since this store was opened. + * + * @return the number of write operations + */ + public long getReadBytes() { + return readBytes.get(); + } + + public boolean isReadOnly() { + return readOnly; + } + + /** + * Get the default retention time for this store in milliseconds. + * + * @return the retention time + */ + public int getDefaultRetentionTime() { + return 45_000; + } + + /** + * Mark the space as in use. + * + * @param pos the position in bytes + * @param length the number of bytes + */ + public void markUsed(long pos, int length) { + freeSpace.markUsed(pos, length); + } + + /** + * Allocate a number of blocks and mark them as used. + * + * @param length the number of bytes to allocate + * @param reservedLow start block index of the reserved area (inclusive) + * @param reservedHigh end block index of the reserved area (exclusive), + * special value -1 means beginning of the infinite free area + * @return the start position in bytes + */ + long allocate(int length, long reservedLow, long reservedHigh) { + return freeSpace.allocate(length, reservedLow, reservedHigh); + } + + /** + * Calculate starting position of the prospective allocation. + * + * @param blocks the number of blocks to allocate + * @param reservedLow start block index of the reserved area (inclusive) + * @param reservedHigh end block index of the reserved area (exclusive), + * special value -1 means beginning of the infinite free area + * @return the starting block index + */ + long predictAllocation(int blocks, long reservedLow, long reservedHigh) { + return freeSpace.predictAllocation(blocks, reservedLow, reservedHigh); + } + + boolean isFragmented() { + return freeSpace.isFragmented(); + } + + /** + * Mark the space as free. + * + * @param pos the position in bytes + * @param length the number of bytes + */ + public void free(long pos, int length) { + freeSpace.free(pos, length); + } + + public int getFillRate() { + return freeSpace.getFillRate(); + } + + /** + * Calculates a prospective fill rate, which store would have after rewrite + * of sparsely populated chunk(s) and evacuation of still live data into a + * new chunk. + * + * @param vacatedBlocks + * number of blocks vacated + * @return prospective fill rate (0 - 100) + */ + public int getProjectedFillRate(int vacatedBlocks) { + return freeSpace.getProjectedFillRate(vacatedBlocks); + } + + long getFirstFree() { + return freeSpace.getFirstFree(); + } + + long getFileLengthInUse() { + return freeSpace.getLastFree(); + } + + /** + * Calculates relative "priority" for chunk to be moved. + * + * @param block where chunk starts + * @return priority, bigger number indicate that chunk need to be moved sooner + */ + int getMovePriority(int block) { + return freeSpace.getMovePriority(block); + } + + long getAfterLastBlock() { + return freeSpace.getAfterLastBlock(); + } + + /** + * Mark the file as empty. + */ + public void clear() { + freeSpace.clear(); + } + + /** + * Get the file name. + * + * @return the file name + */ + public String getFileName() { + return fileName; + } + +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/FreeSpaceBitSet.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/FreeSpaceBitSet.java new file mode 100644 index 000000000..cc92ca7c3 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/FreeSpaceBitSet.java @@ -0,0 +1,341 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.h2.util.MathUtils; + +import java.util.BitSet; + +/** + * A free space bit set. + */ +public class FreeSpaceBitSet { + + private static final boolean DETAILED_INFO = false; + + /** + * The first usable block. + */ + private final int firstFreeBlock; + + /** + * The block size in bytes. + */ + private final int blockSize; + + /** + * The bit set. + */ + private final BitSet set = new BitSet(); + + /** + * Left-shifting register, which holds outcomes of recent allocations. Only + * allocations done in "reuseSpace" mode are recorded here. For example, + * rightmost bit set to 1 means that last allocation failed to find a hole + * big enough, and next bit set to 0 means that previous allocation request + * have found one. + */ + private int failureFlags; + + + /** + * Create a new free space map. + * + * @param firstFreeBlock the first free block + * @param blockSize the block size + */ + public FreeSpaceBitSet(int firstFreeBlock, int blockSize) { + this.firstFreeBlock = firstFreeBlock; + this.blockSize = blockSize; + clear(); + } + + /** + * Reset the list. + */ + public void clear() { + set.clear(); + set.set(0, firstFreeBlock); + } + + /** + * Check whether one of the blocks is in use. + * + * @param pos the position in bytes + * @param length the number of bytes + * @return true if a block is in use + */ + public boolean isUsed(long pos, int length) { + int start = getBlock(pos); + int blocks = getBlockCount(length); + for (int i = start; i < start + blocks; i++) { + if (!set.get(i)) { + return false; + } + } + return true; + } + + /** + * Check whether one of the blocks is free. + * + * @param pos the position in bytes + * @param length the number of bytes + * @return true if a block is free + */ + public boolean isFree(long pos, int length) { + int start = getBlock(pos); + int blocks = getBlockCount(length); + for (int i = start; i < start + blocks; i++) { + if (set.get(i)) { + return false; + } + } + return true; + } + + /** + * Allocate a number of blocks and mark them as used. + * + * @param length the number of bytes to allocate + * @return the start position in bytes + */ + public long allocate(int length) { + return allocate(length, 0, 0); + } + + /** + * Allocate a number of blocks and mark them as used. + * + * @param length the number of bytes to allocate + * @param reservedLow start block index of the reserved area (inclusive) + * @param reservedHigh end block index of the reserved area (exclusive), + * special value -1 means beginning of the infinite free area + * @return the start position in bytes + */ + long allocate(int length, long reservedLow, long reservedHigh) { + return getPos(allocate(getBlockCount(length), (int)reservedLow, (int)reservedHigh, true)); + } + + /** + * Calculate starting position of the prospective allocation. + * + * @param blocks the number of blocks to allocate + * @param reservedLow start block index of the reserved area (inclusive) + * @param reservedHigh end block index of the reserved area (exclusive), + * special value -1 means beginning of the infinite free area + * @return the starting block index + */ + long predictAllocation(int blocks, long reservedLow, long reservedHigh) { + return allocate(blocks, (int)reservedLow, (int)reservedHigh, false); + } + + boolean isFragmented() { + return Integer.bitCount(failureFlags & 0x0F) > 1; + } + + private int allocate(int blocks, int reservedLow, int reservedHigh, boolean allocate) { + int freeBlocksTotal = 0; + for (int i = 0;;) { + int start = set.nextClearBit(i); + int end = set.nextSetBit(start + 1); + int freeBlocks = end - start; + if (end < 0 || freeBlocks >= blocks) { + if ((reservedHigh < 0 || start < reservedHigh) && start + blocks > reservedLow) { // overlap detected + if (reservedHigh < 0) { + start = getAfterLastBlock(); + end = -1; + } else { + i = reservedHigh; + continue; + } + } + assert set.nextSetBit(start) == -1 || set.nextSetBit(start) >= start + blocks : + "Double alloc: " + Integer.toHexString(start) + "/" + Integer.toHexString(blocks) + " " + this; + if (allocate) { + set.set(start, start + blocks); + } else { + failureFlags <<= 1; + if (end < 0 && freeBlocksTotal > 4 * blocks) { + failureFlags |= 1; + } + } + return start; + } + freeBlocksTotal += freeBlocks; + i = end; + } + } + + /** + * Mark the space as in use. + * + * @param pos the position in bytes + * @param length the number of bytes + */ + public void markUsed(long pos, int length) { + int start = getBlock(pos); + int blocks = getBlockCount(length); + assert set.nextSetBit(start) == -1 || set.nextSetBit(start) >= start + blocks : + "Double mark: " + Integer.toHexString(start) + "/" + Integer.toHexString(blocks) + " " + this; + set.set(start, start + blocks); + } + + /** + * Mark the space as free. + * + * @param pos the position in bytes + * @param length the number of bytes + */ + public void free(long pos, int length) { + int start = getBlock(pos); + int blocks = getBlockCount(length); + assert set.nextClearBit(start) >= start + blocks : + "Double free: " + Integer.toHexString(start) + "/" + Integer.toHexString(blocks) + " " + this; + set.clear(start, start + blocks); + } + + private long getPos(int block) { + return (long) block * (long) blockSize; + } + + private int getBlock(long pos) { + return (int) (pos / blockSize); + } + + private int getBlockCount(int length) { + return MathUtils.roundUpInt(length, blockSize) / blockSize; + } + + /** + * Get the fill rate of the space in percent. The value 0 means the space is + * completely free, and 100 means it is completely full. + * + * @return the fill rate (0 - 100) + */ + int getFillRate() { + return getProjectedFillRate(0); + } + + /** + * Calculates a prospective fill rate, which store would have after rewrite + * of sparsely populated chunk(s) and evacuation of still live data into a + * new chunk. + * + * @param vacatedBlocks + * number of blocks vacated as a result of live data evacuation less + * number of blocks in prospective chunk with evacuated live data + * @return prospective fill rate (0 - 100) + */ + int getProjectedFillRate(int vacatedBlocks) { + // it's not bullet-proof against race condition but should be good enough + // to get approximation without holding a store lock + int usedBlocks; + int totalBlocks; + do { + totalBlocks = set.length(); + usedBlocks = set.cardinality(); + } while (totalBlocks != set.length() || usedBlocks > totalBlocks); + usedBlocks -= firstFreeBlock + vacatedBlocks; + totalBlocks -= firstFreeBlock; + return usedBlocks == 0 ? 0 : (int)((100L * usedBlocks + totalBlocks - 1) / totalBlocks); + } + + /** + * Get the position of the first free space. + * + * @return the position. + */ + long getFirstFree() { + return getPos(set.nextClearBit(0)); + } + + /** + * Get the position of the last (infinite) free space. + * + * @return the position. + */ + long getLastFree() { + return getPos(getAfterLastBlock()); + } + + /** + * Get the index of the first block after last occupied one. + * It marks the beginning of the last (infinite) free space. + * + * @return block index + */ + int getAfterLastBlock() { + return set.previousSetBit(set.size() - 1) + 1; + } + + /** + * Calculates relative "priority" for chunk to be moved. + * + * @param block where chunk starts + * @return priority, bigger number indicate that chunk need to be moved sooner + */ + int getMovePriority(int block) { + // The most desirable chunks to move are the ones sitting within + // a relatively short span of occupied blocks which is surrounded + // from both sides by relatively long free spans + int prevEnd = set.previousClearBit(block); + int freeSize; + if (prevEnd < 0) { + prevEnd = firstFreeBlock; + freeSize = 0; + } else { + freeSize = prevEnd - set.previousSetBit(prevEnd); + } + + int nextStart = set.nextClearBit(block); + int nextEnd = set.nextSetBit(nextStart); + if (nextEnd >= 0) { + freeSize += nextEnd - nextStart; + } + return (nextStart - prevEnd - 1) * 1000 / (freeSize + 1); + } + + @Override + public String toString() { + StringBuilder buff = new StringBuilder(); + if (DETAILED_INFO) { + int onCount = 0, offCount = 0; + int on = 0; + for (int i = 0; i < set.length(); i++) { + if (set.get(i)) { + onCount++; + on++; + } else { + offCount++; + } + if ((i & 1023) == 1023) { + buff.append(String.format("%3x", on)).append(' '); + on = 0; + } + } + buff.append('\n') + .append(" on ").append(onCount).append(" off ").append(offCount) + .append(' ').append(100 * onCount / (onCount+offCount)).append("% used "); + } + buff.append('['); + for (int i = 0;;) { + if (i > 0) { + buff.append(", "); + } + int start = set.nextClearBit(i); + buff.append(Integer.toHexString(start)).append('-'); + int end = set.nextSetBit(start + 1); + if (end < 0) { + break; + } + buff.append(Integer.toHexString(end - 1)); + i = end + 1; + } + buff.append(']'); + return buff.toString(); + } +} \ No newline at end of file diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/MVMap.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/MVMap.java new file mode 100644 index 000000000..b449e1d62 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/MVMap.java @@ -0,0 +1,2095 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.type.DataType; +import org.dizitart.no2.mvstore.compat.v1.mvstore.type.ObjectDataType; +import org.dizitart.no2.mvstore.compat.v1.mvstore.type.StringDataType; + +import java.util.AbstractList; +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A stored map. + *

+ * All read and write operations can happen concurrently with all other + * operations, without risk of corruption. + * + * @param the key class + * @param the value class + */ +public class MVMap extends AbstractMap + implements ConcurrentMap +{ + /** + * The store. + */ + public final MVStore store; + + /** + * Reference to the current root page. + */ + private final AtomicReference root; + + private final int id; + private final long createVersion; + private final DataType keyType; + private final DataType valueType; + private final int keysPerPage; + private final boolean singleWriter; + private final K[] keysBuffer; + private final V[] valuesBuffer; + + private final Object lock = new Object(); + private volatile boolean notificationRequested; + + /** + * Whether the map is closed. Volatile so we don't accidentally write to a + * closed map in multithreaded mode. + */ + private volatile boolean closed; + private boolean readOnly; + private boolean isVolatile; + + /** + * This designates the "last stored" version for a store which was + * just open for the first time. + */ + static final long INITIAL_VERSION = -1; + + + protected MVMap(Map config) { + this((MVStore) config.get("store"), + (DataType) config.get("key"), + (DataType) config.get("val"), + DataUtils.readHexInt(config, "id", 0), + DataUtils.readHexLong(config, "createVersion", 0), + new AtomicReference(), + ((MVStore) config.get("store")).getKeysPerPage(), + config.containsKey("singleWriter") && (Boolean) config.get("singleWriter") + ); + setInitialRoot(createEmptyLeaf(), store.getCurrentVersion()); + } + + // constructor for cloneIt() + protected MVMap(MVMap source) { + this(source.store, source.keyType, source.valueType, source.id, source.createVersion, + new AtomicReference<>(source.root.get()), source.keysPerPage, source.singleWriter); + } + + // meta map constructor + MVMap(MVStore store) { + this(store, StringDataType.INSTANCE,StringDataType.INSTANCE, 0, 0, new AtomicReference(), + store.getKeysPerPage(), false); + setInitialRoot(createEmptyLeaf(), store.getCurrentVersion()); + } + + @SuppressWarnings("unchecked") + private MVMap(MVStore store, DataType keyType, DataType valueType, int id, long createVersion, + AtomicReference root, int keysPerPage, boolean singleWriter) { + this.store = store; + this.id = id; + this.createVersion = createVersion; + this.keyType = keyType; + this.valueType = valueType; + this.root = root; + this.keysPerPage = keysPerPage; + this.keysBuffer = singleWriter ? (K[]) new Object[keysPerPage] : null; + this.valuesBuffer = singleWriter ? (V[]) new Object[keysPerPage] : null; + this.singleWriter = singleWriter; + } + + /** + * Clone the current map. + * + * @return clone of this. + */ + protected MVMap cloneIt() { + return new MVMap<>(this); + } + + /** + * Get the metadata key for the root of the given map id. + * + * @param mapId the map id + * @return the metadata key + */ + static String getMapRootKey(int mapId) { + return DataUtils.META_ROOT + Integer.toHexString(mapId); + } + + /** + * Get the metadata key for the given map id. + * + * @param mapId the map id + * @return the metadata key + */ + static String getMapKey(int mapId) { + return DataUtils.META_MAP + Integer.toHexString(mapId); + } + + /** + * Add or replace a key-value pair. + * + * @param key the key (may not be null) + * @param value the value (may not be null) + * @return the old value if the key existed, or null otherwise + */ + @Override + public V put(K key, V value) { + DataUtils.checkArgument(value != null, "The value may not be null"); + return operate(key, value, DecisionMaker.PUT); + } + + /** + * Get the first key, or null if the map is empty. + * + * @return the first key, or null + */ + public final K firstKey() { + return getFirstLast(true); + } + + /** + * Get the first key of this page. + * + * @param p the page + * @return the key, or null + */ + public final K firstKey(Page p) { + return getFirstLast(p, true); + } + + /** + * Get the last key, or null if the map is empty. + * + * @return the last key, or null + */ + public final K lastKey() { + return getFirstLast(false); + } + + /** + * Get the last key of this page. + * + * @param p the page + * @return the key, or null + */ + public final K lastKey(Page p) { + return getFirstLast(p, false); + } + + /** + * Get the key at the given index. + *

+ * This is a O(log(size)) operation. + * + * @param index the index + * @return the key + */ + public final K getKey(long index) { + if (index < 0 || index >= sizeAsLong()) { + return null; + } + Page p = getRootPage(); + long offset = 0; + while (true) { + if (p.isLeaf()) { + if (index >= offset + p.getKeyCount()) { + return null; + } + @SuppressWarnings("unchecked") + K key = (K) p.getKey((int) (index - offset)); + return key; + } + int i = 0, size = getChildPageCount(p); + for (; i < size; i++) { + long c = p.getCounts(i); + if (index < c + offset) { + break; + } + offset += c; + } + if (i == size) { + return null; + } + p = p.getChildPage(i); + } + } + + /** + * Get the key list. The list is a read-only representation of all keys. + *

+ * The get and indexOf methods are O(log(size)) operations. The result of + * indexOf is cast to an int. + * + * @return the key list + */ + public final List keyList() { + return new AbstractList() { + + @Override + public K get(int index) { + return getKey(index); + } + + @Override + public int size() { + return MVMap.this.size(); + } + + @Override + @SuppressWarnings("unchecked") + public int indexOf(Object key) { + return (int) getKeyIndex((K) key); + } + + }; + } + + /** + * Get the index of the given key in the map. + *

+ * This is a O(log(size)) operation. + *

+ * If the key was found, the returned value is the index in the key array. + * If not found, the returned value is negative, where -1 means the provided + * key is smaller than any keys. See also Arrays.binarySearch. + * + * @param key the key + * @return the index + */ + public final long getKeyIndex(K key) { + Page p = getRootPage(); + if (p.getTotalCount() == 0) { + return -1; + } + long offset = 0; + while (true) { + int x = p.binarySearch(key); + if (p.isLeaf()) { + if (x < 0) { + offset = -offset; + } + return offset + x; + } + if (x++ < 0) { + x = -x; + } + for (int i = 0; i < x; i++) { + offset += p.getCounts(i); + } + p = p.getChildPage(x); + } + } + + /** + * Get the first (lowest) or last (largest) key. + * + * @param first whether to retrieve the first key + * @return the key, or null if the map is empty + */ + private K getFirstLast(boolean first) { + Page p = getRootPage(); + return getFirstLast(p, first); + } + + @SuppressWarnings("unchecked") + private K getFirstLast(Page p, boolean first) { + if (p.getTotalCount() == 0) { + return null; + } + while (true) { + if (p.isLeaf()) { + return (K) p.getKey(first ? 0 : p.getKeyCount() - 1); + } + p = p.getChildPage(first ? 0 : getChildPageCount(p) - 1); + } + } + + /** + * Get the smallest key that is larger than the given key, or null if no + * such key exists. + * + * @param key the key + * @return the result + */ + public final K higherKey(K key) { + return getMinMax(key, false, true); + } + + /** + * Get the smallest key that is larger than the given key, for the given + * root page, or null if no such key exists. + * + * @param p the root page + * @param key the key + * @return the result + */ + public final K higherKey(Page p, K key) { + return getMinMax(p, key, false, true); + } + + /** + * Get the smallest key that is larger or equal to this key. + * + * @param key the key + * @return the result + */ + public final K ceilingKey(K key) { + return getMinMax(key, false, false); + } + + /** + * Get the smallest key that is larger or equal to this key, for the given root page. + * + * @param p the root page + * @param key the key + * @return the result + */ + public final K ceilingKey(Page p, K key) { + return getMinMax(p, key, false, false); + } + + /** + * Get the largest key that is smaller or equal to this key. + * + * @param key the key + * @return the result + */ + public final K floorKey(K key) { + return getMinMax(key, true, false); + } + + /** + * Get the largest key that is smaller or equal to this key, for the given root page. + * + * @param p the root page + * @param key the key + * @return the result + */ + public final K floorKey(Page p, K key) { + return getMinMax(p, key, true, false); + } + + /** + * Get the largest key that is smaller than the given key, or null if no + * such key exists. + * + * @param key the key + * @return the result + */ + public final K lowerKey(K key) { + return getMinMax(key, true, true); + } + + /** + * Get the largest key that is smaller than the given key, for the given + * root page, or null if no such key exists. + * + * @param p the root page + * @param key the key + * @return the result + */ + public final K lowerKey(Page p, K key) { + return getMinMax(p, key, true, true); + } + + /** + * Get the smallest or largest key using the given bounds. + * + * @param key the key + * @param min whether to retrieve the smallest key + * @param excluding if the given upper/lower bound is exclusive + * @return the key, or null if no such key exists + */ + private K getMinMax(K key, boolean min, boolean excluding) { + return getMinMax(getRootPage(), key, min, excluding); + } + + @SuppressWarnings("unchecked") + private K getMinMax(Page p, K key, boolean min, boolean excluding) { + int x = p.binarySearch(key); + if (p.isLeaf()) { + if (x < 0) { + x = -x - (min ? 2 : 1); + } else if (excluding) { + x += min ? -1 : 1; + } + if (x < 0 || x >= p.getKeyCount()) { + return null; + } + return (K) p.getKey(x); + } + if (x++ < 0) { + x = -x; + } + while (true) { + if (x < 0 || x >= getChildPageCount(p)) { + return null; + } + K k = getMinMax(p.getChildPage(x), key, min, excluding); + if (k != null) { + return k; + } + x += min ? -1 : 1; + } + } + + + /** + * Get the value for the given key, or null if not found. + * + * @param key the key + * @return the value, or null if not found + * @throws ClassCastException if type of the specified key is not compatible with this map + */ + @Override + public final V get(Object key) { + return get(getRootPage(), key); + } + + /** + * Get the value for the given key from a snapshot, or null if not found. + * + * @param p the root of a snapshot + * @param key the key + * @return the value, or null if not found + * @throws ClassCastException if type of the specified key is not compatible with this map + */ + @SuppressWarnings("unchecked") + public V get(Page p, Object key) { + return (V) Page.get(p, key); + } + + @Override + public final boolean containsKey(Object key) { + return get(key) != null; + } + + /** + * Remove all entries. + */ + @Override + public void clear() { + clearIt(); + } + + /** + * Remove all entries and return the root reference. + * + * @return the new root reference + */ + RootReference clearIt() { + Page emptyRootPage = createEmptyLeaf(); + int attempt = 0; + while (true) { + RootReference rootReference = flushAndGetRoot(); + if (rootReference.getTotalCount() == 0) { + return rootReference; + } + boolean locked = rootReference.isLockedByCurrentThread(); + if (!locked) { + if (attempt++ == 0) { + beforeWrite(); + } else if (attempt > 3 || rootReference.isLocked()) { + rootReference = lockRoot(rootReference, attempt); + locked = true; + } + } + Page rootPage = rootReference.root; + long version = rootReference.version; + try { + if (!locked) { + rootReference = rootReference.updateRootPage(emptyRootPage, attempt); + if (rootReference == null) { + continue; + } + } + store.registerUnsavedMemory(rootPage.removeAllRecursive(version)); + rootPage = emptyRootPage; + return rootReference; + } finally { + if(locked) { + unlockRoot(rootPage); + } + } + } + } + + /** + * Close the map. Accessing the data is still possible (to allow concurrent + * reads), but it is marked as closed. + */ + final void close() { + closed = true; + } + + public final boolean isClosed() { + return closed; + } + + /** + * Remove a key-value pair, if the key exists. + * + * @param key the key (may not be null) + * @return the old value if the key existed, or null otherwise + * @throws ClassCastException if type of the specified key is not compatible with this map + */ + @Override + @SuppressWarnings("unchecked") + public V remove(Object key) { + return operate((K)key, null, DecisionMaker.REMOVE); + } + + /** + * Add a key-value pair if it does not yet exist. + * + * @param key the key (may not be null) + * @param value the new value + * @return the old value if the key existed, or null otherwise + */ + @Override + public final V putIfAbsent(K key, V value) { + return operate(key, value, DecisionMaker.IF_ABSENT); + } + + /** + * Remove a key-value pair if the value matches the stored one. + * + * @param key the key (may not be null) + * @param value the expected value + * @return true if the item was removed + */ + @SuppressWarnings("unchecked") + @Override + public boolean remove(Object key, Object value) { + EqualsDecisionMaker decisionMaker = new EqualsDecisionMaker<>(valueType, (V)value); + operate((K)key, null, decisionMaker); + return decisionMaker.getDecision() != Decision.ABORT; + } + + /** + * Check whether the two values are equal. + * + * @param a the first value + * @param b the second value + * @param datatype to use for comparison + * @return true if they are equal + */ + static boolean areValuesEqual(DataType datatype, Object a, Object b) { + return a == b + || a != null && b != null && datatype.compare(a, b) == 0; + } + + /** + * Replace a value for an existing key, if the value matches. + * + * @param key the key (may not be null) + * @param oldValue the expected value + * @param newValue the new value + * @return true if the value was replaced + */ + @Override + public final boolean replace(K key, V oldValue, V newValue) { + EqualsDecisionMaker decisionMaker = new EqualsDecisionMaker<>(valueType, oldValue); + V result = operate(key, newValue, decisionMaker); + boolean res = decisionMaker.getDecision() != Decision.ABORT; + assert !res || areValuesEqual(valueType, oldValue, result) : oldValue + " != " + result; + return res; + } + + private boolean rewrite(K key) { + ContainsDecisionMaker decisionMaker = new ContainsDecisionMaker<>(); + V result = operate(key, null, decisionMaker); + boolean res = decisionMaker.getDecision() != Decision.ABORT; + assert res == (result != null); + return res; + } + + /** + * Replace a value for an existing key. + * + * @param key the key (may not be null) + * @param value the new value + * @return the old value, if the value was replaced, or null + */ + @Override + public final V replace(K key, V value) { + return operate(key, value, DecisionMaker.IF_PRESENT); + } + + /** + * Compare two keys. + * + * @param a the first key + * @param b the second key + * @return -1 if the first key is smaller, 1 if bigger, 0 if equal + */ + final int compare(Object a, Object b) { + return keyType.compare(a, b); + } + + /** + * Get the key type. + * + * @return the key type + */ + public final DataType getKeyType() { + return keyType; + } + + /** + * Get the value type. + * + * @return the value type + */ + public final DataType getValueType() { + return valueType; + } + + boolean isSingleWriter() { + return singleWriter; + } + + /** + * Read a page. + * + * @param pos the position of the page + * @return the page + */ + final Page readPage(long pos) { + return store.readPage(this, pos); + } + + /** + * Set the position of the root page. + * @param rootPos the position, 0 for empty + * @param version to set for this map + * + */ + final void setRootPos(long rootPos, long version) { + Page root = readOrCreateRootPage(rootPos); + setInitialRoot(root, version); + setWriteVersion(store.getCurrentVersion()); + } + + private Page readOrCreateRootPage(long rootPos) { + Page root = rootPos == 0 ? createEmptyLeaf() : readPage(rootPos); + return root; + } + + /** + * Iterate over a number of keys. + * + * @param from the first key to return + * @return the iterator + */ + public final Iterator keyIterator(K from) { + return new Cursor(getRootPage(), from); + } + + /** + * Re-write any pages that belong to one of the chunks in the given set. + * + * @param set the set of chunk ids + * @return number of pages actually re-written + */ + final int rewrite(Set set) { + if (!singleWriter) { + return rewrite(getRootPage(), set); + } + RootReference rootReference = lockRoot(getRoot(), 1); + int appendCounter = rootReference.getAppendCounter(); + try { + if (appendCounter > 0) { + rootReference = flushAppendBuffer(rootReference, true); + assert rootReference.getAppendCounter() == 0; + } + int res = rewrite(rootReference.root, set); + return res; + } finally { + unlockRoot(); + } + } + + private int rewrite(Page p, Set set) { + if (p.isLeaf()) { + long pos = p.getPos(); + int chunkId = DataUtils.getPageChunkId(pos); + if (!set.contains(chunkId)) { + return 0; + } + assert p.getKeyCount() > 0; + return rewritePage(p) ? 1 : 0; + } + int writtenPageCount = 0; + for (int i = 0; i < getChildPageCount(p); i++) { + long childPos = p.getChildPagePos(i); + if (childPos != 0 && DataUtils.getPageType(childPos) == DataUtils.PAGE_TYPE_LEAF) { + // we would need to load the page, and it's a leaf: + // only do that if it's within the set of chunks we are + // interested in + int chunkId = DataUtils.getPageChunkId(childPos); + if (!set.contains(chunkId)) { + continue; + } + } + writtenPageCount += rewrite(p.getChildPage(i), set); + } + if (writtenPageCount == 0) { + long pos = p.getPos(); + int chunkId = DataUtils.getPageChunkId(pos); + if (set.contains(chunkId)) { + // an inner node page that is in one of the chunks, + // but only points to chunks that are not in the set: + // if no child was changed, we need to do that now + // (this is not needed if anyway one of the children + // was changed, as this would have updated this + // page as well) + while (!p.isLeaf()) { + p = p.getChildPage(0); + } + if (rewritePage(p)) { + writtenPageCount = 1; + } + } + } + return writtenPageCount; + } + + private boolean rewritePage(Page p) { + @SuppressWarnings("unchecked") + K key = (K) p.getKey(0); + if (!isClosed()) { + return rewrite(key); + } + return true; + } + + /** + * Get a cursor to iterate over a number of keys and values. + * + * @param from the first key to return + * @return the cursor + */ + public final Cursor cursor(K from) { + return new Cursor<>(getRootPage(), from); + } + + @Override + public final Set> entrySet() { + final Page root = this.getRootPage(); + return new AbstractSet>() { + + @Override + public Iterator> iterator() { + final Cursor cursor = new Cursor<>(root, null); + return new Iterator>() { + + @Override + public boolean hasNext() { + return cursor.hasNext(); + } + + @Override + public Entry next() { + K k = cursor.next(); + return new SimpleImmutableEntry<>(k, cursor.getValue()); + } + + @Override + public void remove() { + throw DataUtils.newUnsupportedOperationException( + "Removing is not supported"); + } + }; + + } + + @Override + public int size() { + return MVMap.this.size(); + } + + @Override + public boolean contains(Object o) { + return MVMap.this.containsKey(o); + } + + }; + + } + + @Override + public Set keySet() { + final Page root = getRootPage(); + return new AbstractSet() { + + @Override + public Iterator iterator() { + return new Cursor(root, null); + } + + @Override + public int size() { + return MVMap.this.size(); + } + + @Override + public boolean contains(Object o) { + return MVMap.this.containsKey(o); + } + + }; + } + + /** + * Get the map name. + * + * @return the name + */ + public final String getName() { + return store.getMapName(id); + } + + public final MVStore getStore() { + return store; + } + + protected final boolean isPersistent() { + return store.getFileStore() != null && !isVolatile; + } + + /** + * Get the map id. Please note the map id may be different after compacting + * a store. + * + * @return the map id + */ + public final int getId() { + return id; + } + + /** + * The current root page (may not be null). + * + * @return the root page + */ + public final Page getRootPage() { + return flushAndGetRoot().root; + } + + public RootReference getRoot() { + return root.get(); + } + + /** + * Get the root reference, flushing any current append buffer. + * + * @return current root reference + */ + public RootReference flushAndGetRoot() { + RootReference rootReference = getRoot(); + if (singleWriter && rootReference.getAppendCounter() > 0) { + return flushAppendBuffer(rootReference, true); + } + return rootReference; + } + + /** + * Set the initial root. + * + * @param rootPage root page + * @param version initial version + */ + final void setInitialRoot(Page rootPage, long version) { + root.set(new RootReference(rootPage, version)); + } + + /** + * Compare and set the root reference. + * + * @param expectedRootReference the old (expected) + * @param updatedRootReference the new + * @return whether updating worked + */ + final boolean compareAndSetRoot(RootReference expectedRootReference, RootReference updatedRootReference) { + return root.compareAndSet(expectedRootReference, updatedRootReference); + } + + /** + * Rollback to the given version. + * + * @param version the version + */ + final void rollbackTo(long version) { + // check if the map was removed and re-created later ? + if (version > createVersion) { + rollbackRoot(version); + } + } + + /** + * Roll the root back to the specified version. + * + * @param version to rollback to + * @return true if rollback was a success, false if there was not enough in-memory history + */ + boolean rollbackRoot(long version) + { + RootReference rootReference = flushAndGetRoot(); + RootReference previous; + while (rootReference.version >= version && (previous = rootReference.previous) != null) { + if (root.compareAndSet(rootReference, previous)) { + rootReference = previous; + closed = false; + } + } + setWriteVersion(version); + return rootReference.version < version; + } + + /** + * Use the new root page from now on. + * @param expectedRootReference expected current root reference + * @param newRootPage the new root page + * @param attemptUpdateCounter how many attempt (including current) + * were made to update root + * @return new RootReference or null if update failed + */ + protected static boolean updateRoot(RootReference expectedRootReference, Page newRootPage, + int attemptUpdateCounter) { + return expectedRootReference.updateRootPage(newRootPage, attemptUpdateCounter) != null; + } + + /** + * Forget those old versions that are no longer needed. + * @param rootReference to inspect + */ + private void removeUnusedOldVersions(RootReference rootReference) { + rootReference.removeUnusedOldVersions(store.getOldestVersionToKeep()); + } + + public final boolean isReadOnly() { + return readOnly; + } + + /** + * Set the volatile flag of the map. + * + * @param isVolatile the volatile flag + */ + public final void setVolatile(boolean isVolatile) { + this.isVolatile = isVolatile; + } + + /** + * Whether this is volatile map, meaning that changes + * are not persisted. By default (even if the store is not persisted), + * maps are not volatile. + * + * @return whether this map is volatile + */ + public final boolean isVolatile() { + return isVolatile; + } + + /** + * This method is called before writing to the map. The default + * implementation checks whether writing is allowed, and tries + * to detect concurrent modification. + * + * @throws UnsupportedOperationException if the map is read-only, + * or if another thread is concurrently writing + */ + protected final void beforeWrite() { + assert !getRoot().isLockedByCurrentThread() : getRoot(); + if (closed) { + int id = getId(); + String mapName = store.getMapName(id); + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_CLOSED, "Map {0}({1}) is closed. {2}", mapName, id, store.getPanicException()); + } + if (readOnly) { + throw DataUtils.newUnsupportedOperationException( + "This map is read-only"); + } + store.beforeWrite(this); + } + + @Override + public final int hashCode() { + return id; + } + + @Override + public final boolean equals(Object o) { + return this == o; + } + + /** + * Get the number of entries, as a integer. {@link Integer#MAX_VALUE} is + * returned if there are more than this entries. + * + * @return the number of entries, as an integer + * @see #sizeAsLong() + */ + @Override + public final int size() { + long size = sizeAsLong(); + return size > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) size; + } + + /** + * Get the number of entries, as a long. + * + * @return the number of entries + */ + public final long sizeAsLong() { + return getRoot().getTotalCount(); + } + + @Override + public boolean isEmpty() { + return sizeAsLong() == 0; + } + + public final long getCreateVersion() { + return createVersion; + } + + /** + * Open an old version for the given map. + * It will restore map at last known state of the version specified. + * (at the point right before the commit() call, which advanced map to the next version) + * Map is opened in read-only mode. + * + * @param version the version + * @return the map + */ + public final MVMap openVersion(long version) { + if (readOnly) { + throw DataUtils.newUnsupportedOperationException( + "This map is read-only; need to call " + + "the method on the writable map"); + } + DataUtils.checkArgument(version >= createVersion, + "Unknown version {0}; this map was created in version is {1}", + version, createVersion); + RootReference rootReference = flushAndGetRoot(); + removeUnusedOldVersions(rootReference); + RootReference previous; + while ((previous = rootReference.previous) != null && previous.version >= version) { + rootReference = previous; + } + if (previous == null && version < store.getOldestVersionToKeep()) { + throw DataUtils.newIllegalArgumentException("Unknown version {0}", version); + } + MVMap m = openReadOnly(rootReference.root, version); + assert m.getVersion() <= version : m.getVersion() + " <= " + version; + return m; + } + + /** + * Open a copy of the map in read-only mode. + * + * @param rootPos position of the root page + * @param version to open + * @return the opened map + */ + final MVMap openReadOnly(long rootPos, long version) { + Page root = readOrCreateRootPage(rootPos); + return openReadOnly(root, version); + } + + private MVMap openReadOnly(Page root, long version) { + MVMap m = cloneIt(); + m.readOnly = true; + m.setInitialRoot(root, version); + return m; + } + + /** + * Get version of the map, which is the version of the store, + * at the moment when map was modified last time. + * + * @return version + */ + public final long getVersion() { + return getRoot().getVersion(); + } + + /** + * Does the root have changes since the specified version? + * + * @param version root version + * @return true if has changes + */ + final boolean hasChangesSince(long version) { + return getRoot().hasChangesSince(version); + } + + /** + * Get the child page count for this page. This is to allow another map + * implementation to override the default, in case the last child is not to + * be used. + * + * @param p the page + * @return the number of direct children + */ + protected int getChildPageCount(Page p) { + return p.getRawChildPageCount(); + } + + /** + * Get the map type. When opening an existing map, the map type must match. + * + * @return the map type + */ + public String getType() { + return null; + } + + /** + * Get the map metadata as a string. + * + * @param name the map name (or null) + * @return the string + */ + protected String asString(String name) { + StringBuilder buff = new StringBuilder(); + if (name != null) { + DataUtils.appendMap(buff, "name", name); + } + if (createVersion != 0) { + DataUtils.appendMap(buff, "createVersion", createVersion); + } + String type = getType(); + if (type != null) { + DataUtils.appendMap(buff, "type", type); + } + return buff.toString(); + } + + final RootReference setWriteVersion(long writeVersion) { + int attempt = 0; + while(true) { + RootReference rootReference = flushAndGetRoot(); + if(rootReference.version >= writeVersion) { + return rootReference; + } else if (isClosed()) { + // map was closed a while back and can not possibly be in use by now + // it's time to remove it completely from the store (it was anonymous already) + if (rootReference.getVersion() + 1 < store.getOldestVersionToKeep()) { + store.deregisterMapRoot(id); + return null; + } + } + + RootReference lockedRootReference = null; + if (++attempt > 3 || rootReference.isLocked()) { + lockedRootReference = lockRoot(rootReference, attempt); + rootReference = flushAndGetRoot(); + } + + try { + rootReference = rootReference.tryUnlockAndUpdateVersion(writeVersion, attempt); + if (rootReference != null) { + lockedRootReference = null; + removeUnusedOldVersions(rootReference); + return rootReference; + } + } finally { + if (lockedRootReference != null) { + unlockRoot(); + } + } + } + } + + /** + * Create empty leaf node page. + * + * @return new page + */ + protected Page createEmptyLeaf() { + return Page.createEmptyLeaf(this); + } + + /** + * Create empty internal node page. + * + * @return new page + */ + protected Page createEmptyNode() { + return Page.createEmptyNode(this); + } + + /** + * Copy a map. All pages are copied. + * + * @param sourceMap the source map + */ + final void copyFrom(MVMap sourceMap) { + MVStore.TxCounter txCounter = store.registerVersionUsage(); + try { + beforeWrite(); + copy(sourceMap.getRootPage(), null, 0); + } finally { + store.deregisterVersionUsage(txCounter); + } + } + + private void copy(Page source, Page parent, int index) { + Page target = source.copy(this); + if (parent == null) { + setInitialRoot(target, INITIAL_VERSION); + } else { + parent.setChild(index, target); + } + if (!source.isLeaf()) { + for (int i = 0; i < getChildPageCount(target); i++) { + if (source.getChildPagePos(i) != 0) { + // position 0 means no child + // (for example the last entry of an r-tree node) + // (the MVMap is also used for r-trees for compacting) + copy(source.getChildPage(i), target, i); + } + } + target.setComplete(); + } + store.registerUnsavedMemory(target.getMemory()); + if (store.isSaveNeeded()) { + store.commit(); + } + } + + /** + * If map was used in append mode, this method will ensure that append buffer + * is flushed - emptied with all entries inserted into map as a new leaf. + * @param rootReference current RootReference + * @param fullFlush whether buffer should be completely flushed, + * otherwise just a single empty slot is required + * @return potentially updated RootReference + */ + private RootReference flushAppendBuffer(RootReference rootReference, boolean fullFlush) { + boolean preLocked = rootReference.isLockedByCurrentThread(); + boolean locked = preLocked; + int keysPerPage = store.getKeysPerPage(); + try { + IntValueHolder unsavedMemoryHolder = new IntValueHolder(); + int attempt = 0; + int keyCount; + int availabilityThreshold = fullFlush ? 0 : keysPerPage - 1; + while ((keyCount = rootReference.getAppendCounter()) > availabilityThreshold) { + if (!locked) { + // instead of just calling lockRoot() we loop here and check if someone else + // already flushed the buffer, then we don't need a lock + rootReference = tryLock(rootReference, ++attempt); + if (rootReference == null) { + rootReference = getRoot(); + continue; + } + locked = true; + } + + Page rootPage = rootReference.root; + long version = rootReference.version; + CursorPos pos = rootPage.getAppendCursorPos(null); + assert pos != null; + assert pos.index < 0 : pos.index; + int index = -pos.index - 1; + assert index == pos.page.getKeyCount() : index + " != " + pos.page.getKeyCount(); + Page p = pos.page; + CursorPos tip = pos; + pos = pos.parent; + + int remainingBuffer = 0; + Page page = null; + int available = keysPerPage - p.getKeyCount(); + if (available > 0) { + p = p.copy(); + if (keyCount <= available) { + p.expand(keyCount, keysBuffer, valuesBuffer); + } else { + p.expand(available, keysBuffer, valuesBuffer); + keyCount -= available; + if (fullFlush) { + Object[] keys = new Object[keyCount]; + Object[] values = new Object[keyCount]; + System.arraycopy(keysBuffer, available, keys, 0, keyCount); + System.arraycopy(valuesBuffer, available, values, 0, keyCount); + page = Page.createLeaf(this, keys, values, 0); + } else { + System.arraycopy(keysBuffer, available, keysBuffer, 0, keyCount); + System.arraycopy(valuesBuffer, available, valuesBuffer, 0, keyCount); + remainingBuffer = keyCount; + } + } + } else { + tip = tip.parent; + page = Page.createLeaf(this, + Arrays.copyOf(keysBuffer, keyCount), + Arrays.copyOf(valuesBuffer, keyCount), + 0); + } + + unsavedMemoryHolder.value = 0; + if (page != null) { + assert page.map == this; + assert page.getKeyCount() > 0; + Object key = page.getKey(0); + unsavedMemoryHolder.value += page.getMemory(); + while (true) { + if (pos == null) { + if (p.getKeyCount() == 0) { + p = page; + } else { + Object[] keys = new Object[]{key}; + Page.PageReference[] children = new Page.PageReference[]{ + new Page.PageReference(p), + new Page.PageReference(page)}; + unsavedMemoryHolder.value += p.getMemory(); + p = Page.createNode(this, keys, children, p.getTotalCount() + page.getTotalCount(), 0); + } + break; + } + Page c = p; + p = pos.page; + index = pos.index; + pos = pos.parent; + p = p.copy(); + p.setChild(index, page); + p.insertNode(index, key, c); + keyCount = p.getKeyCount(); + int at = keyCount - (p.isLeaf() ? 1 : 2); + if (keyCount <= keysPerPage && + (p.getMemory() < store.getMaxPageSize() || at <= 0)) { + break; + } + key = p.getKey(at); + page = p.split(at); + unsavedMemoryHolder.value += p.getMemory() + page.getMemory(); + } + } + p = replacePage(pos, p, unsavedMemoryHolder); + rootReference = rootReference.updatePageAndLockedStatus(p, preLocked || isPersistent(), + remainingBuffer); + if (rootReference != null) { + // should always be the case, except for spurious failure? + locked = preLocked || isPersistent(); + if (isPersistent() && tip != null) { + store.registerUnsavedMemory(unsavedMemoryHolder.value + tip.processRemovalInfo(version)); + } + assert rootReference.getAppendCounter() <= availabilityThreshold; + break; + } + rootReference = getRoot(); + } + } finally { + if (locked && !preLocked) { + rootReference = unlockRoot(); + } + } + return rootReference; + } + + private static Page replacePage(CursorPos path, Page replacement, IntValueHolder unsavedMemoryHolder) { + int unsavedMemory = replacement.isSaved() ? 0 : replacement.getMemory(); + while (path != null) { + Page parent = path.page; + // condition below should always be true, but older versions (up to 1.4.197) + // may create single-childed (with no keys) internal nodes, which we skip here + if (parent.getKeyCount() > 0) { + Page child = replacement; + replacement = parent.copy(); + replacement.setChild(path.index, child); + unsavedMemory += replacement.getMemory(); + } + path = path.parent; + } + unsavedMemoryHolder.value += unsavedMemory; + return replacement; + } + + /** + * Appends entry to this map. this method is NOT thread safe and can not be used + * neither concurrently, nor in combination with any method that updates this map. + * Non-updating method may be used concurrently, but latest appended values + * are not guaranteed to be visible. + * @param key should be higher in map's order than any existing key + * @param value to be appended + */ + public void append(K key, V value) { + if (singleWriter) { + beforeWrite(); + RootReference rootReference = lockRoot(getRoot(), 1); + int appendCounter = rootReference.getAppendCounter(); + try { + if (appendCounter >= keysPerPage) { + rootReference = flushAppendBuffer(rootReference, false); + appendCounter = rootReference.getAppendCounter(); + assert appendCounter < keysPerPage; + } + keysBuffer[appendCounter] = key; + valuesBuffer[appendCounter] = value; + ++appendCounter; + } finally { + unlockRoot(appendCounter); + } + } else { + put(key, value); + } + } + + /** + * Removes last entry from this map. this method is NOT thread safe and can not be used + * neither concurrently, nor in combination with any method that updates this map. + * Non-updating method may be used concurrently, but latest removal may not be visible. + */ + public void trimLast() { + if (singleWriter) { + RootReference rootReference = getRoot(); + int appendCounter = rootReference.getAppendCounter(); + boolean useRegularRemove = appendCounter == 0; + if (!useRegularRemove) { + rootReference = lockRoot(rootReference, 1); + try { + appendCounter = rootReference.getAppendCounter(); + useRegularRemove = appendCounter == 0; + if (!useRegularRemove) { + --appendCounter; + } + } finally { + unlockRoot(appendCounter); + } + } + if (useRegularRemove) { + Page lastLeaf = rootReference.root.getAppendCursorPos(null).page; + assert lastLeaf.isLeaf(); + assert lastLeaf.getKeyCount() > 0; + Object key = lastLeaf.getKey(lastLeaf.getKeyCount() - 1); + remove(key); + } + } else { + remove(lastKey()); + } + } + + @Override + public final String toString() { + return asString(null); + } + + /** + * A builder for maps. + * + * @param the map type + * @param the key type + * @param the value type + */ + public interface MapBuilder, K, V> { + + /** + * Create a new map of the given type. + * @param store which will own this map + * @param config configuration + * + * @return the map + */ + M create(MVStore store, Map config); + + DataType getKeyType(); + + DataType getValueType(); + + void setKeyType(DataType dataType); + + void setValueType(DataType dataType); + + } + + /** + * A builder for this class. + * + * @param the key type + * @param the value type + */ + public abstract static class BasicBuilder, K, V> implements MapBuilder { + + private DataType keyType; + private DataType valueType; + + /** + * Create a new builder with the default key and value data types. + */ + protected BasicBuilder() { + // ignore + } + + @Override + public DataType getKeyType() { + return keyType; + } + + @Override + public DataType getValueType() { + return valueType; + } + + @Override + public void setKeyType(DataType keyType) { + this.keyType = keyType; + } + + @Override + public void setValueType(DataType valueType) { + this.valueType = valueType; + } + + /** + * Set the key data type. + * + * @param keyType the key type + * @return this + */ + public BasicBuilder keyType(DataType keyType) { + this.keyType = keyType; + return this; + } + + /** + * Set the value data type. + * + * @param valueType the value type + * @return this + */ + public BasicBuilder valueType(DataType valueType) { + this.valueType = valueType; + return this; + } + + @Override + public M create(MVStore store, Map config) { + if (getKeyType() == null) { + setKeyType(new ObjectDataType()); + } + if (getValueType() == null) { + setValueType(new ObjectDataType()); + } + DataType keyType = getKeyType(); + DataType valueType = getValueType(); + config.put("store", store); + config.put("key", keyType); + config.put("val", valueType); + return create(config); + } + + /** + * Create map from config. + * @param config config map + * @return new map + */ + protected abstract M create(Map config); + + } + + /** + * A builder for this class. + * + * @param the key type + * @param the value type + */ + public static class Builder extends BasicBuilder, K, V> { + private boolean singleWriter; + + public Builder() {} + + @Override + public Builder keyType(DataType dataType) { + setKeyType(dataType); + return this; + } + + @Override + public Builder valueType(DataType dataType) { + setValueType(dataType); + return this; + } + + /** + * Set up this Builder to produce MVMap, which can be used in append mode + * by a single thread. + * @see MVMap#append(Object, Object) + * @return this Builder for chained execution + */ + public Builder singleWriter() { + singleWriter = true; + return this; + } + + @Override + protected MVMap create(Map config) { + config.put("singleWriter", singleWriter); + Object type = config.get("type"); + if(type == null || type.equals("rtree")) { + return new MVMap<>(config); + } + throw new IllegalArgumentException("Incompatible map type"); + } + } + + public enum Decision { ABORT, REMOVE, PUT, REPEAT } + + /** + * Class DecisionMaker provides callback interface (and should become a such in Java 8) + * for MVMap.operate method. + * It provides control logic to make a decision about how to proceed with update + * at the point in execution when proper place and possible existing value + * for insert/update/delete key is found. + * Revised value for insert/update is also provided based on original input value + * and value currently existing in the map. + * + * @param value type of the map + */ + public abstract static class DecisionMaker + { + /** + * Decision maker for transaction rollback. + */ + public static final DecisionMaker DEFAULT = new DecisionMaker() { + @Override + public Decision decide(Object existingValue, Object providedValue) { + return providedValue == null ? Decision.REMOVE : Decision.PUT; + } + + @Override + public String toString() { + return "default"; + } + }; + + /** + * Decision maker for put(). + */ + public static final DecisionMaker PUT = new DecisionMaker() { + @Override + public Decision decide(Object existingValue, Object providedValue) { + return Decision.PUT; + } + + @Override + public String toString() { + return "put"; + } + }; + + /** + * Decision maker for remove(). + */ + public static final DecisionMaker REMOVE = new DecisionMaker() { + @Override + public Decision decide(Object existingValue, Object providedValue) { + return Decision.REMOVE; + } + + @Override + public String toString() { + return "remove"; + } + }; + + /** + * Decision maker for putIfAbsent() key/value. + */ + static final DecisionMaker IF_ABSENT = new DecisionMaker() { + @Override + public Decision decide(Object existingValue, Object providedValue) { + return existingValue == null ? Decision.PUT : Decision.ABORT; + } + + @Override + public String toString() { + return "if_absent"; + } + }; + + /** + * Decision maker for replace(). + */ + static final DecisionMaker IF_PRESENT= new DecisionMaker() { + @Override + public Decision decide(Object existingValue, Object providedValue) { + return existingValue != null ? Decision.PUT : Decision.ABORT; + } + + @Override + public String toString() { + return "if_present"; + } + }; + + /** + * Makes a decision about how to proceed with the update. + * @param existingValue value currently exists in the map + * @param providedValue original input value + * @return PUT if a new value need to replace existing one or + * new value to be inserted if there is none + * REMOVE if existing value should be deleted + * ABORT if update operation should be aborted + */ + public abstract Decision decide(V existingValue, V providedValue); + + /** + * Provides revised value for insert/update based on original input value + * and value currently existing in the map. + * This method is only invoked after call to decide(), if it returns PUT. + * @param existingValue value currently exists in the map + * @param providedValue original input value + * @param value type + * @return value to be used by insert/update + */ + public T selectValue(T existingValue, T providedValue) { + return providedValue; + } + + /** + * Resets internal state (if any) of a this DecisionMaker to it's initial state. + * This method is invoked whenever concurrent update failure is encountered, + * so we can re-start update process. + */ + public void reset() {} + } + + /** + * Add, replace or remove a key-value pair. + * + * @param key the key (may not be null) + * @param value new value, it may be null when removal is intended + * @param decisionMaker command object to make choices during transaction. + * @return previous value, if mapping for that key existed, or null otherwise + */ + @SuppressWarnings("unchecked") + public V operate(K key, V value, DecisionMaker decisionMaker) { + IntValueHolder unsavedMemoryHolder = new IntValueHolder(); + int attempt = 0; + while(true) { + RootReference rootReference = flushAndGetRoot(); + boolean locked = rootReference.isLockedByCurrentThread(); + if (!locked) { + if (attempt++ == 0) { + beforeWrite(); + } else if (attempt > 3 || rootReference.isLocked()) { + rootReference = lockRoot(rootReference, attempt); + locked = true; + } + } + Page rootPage = rootReference.root; + long version = rootReference.version; + CursorPos tip; + V result; + unsavedMemoryHolder.value = 0; + try { + CursorPos pos = CursorPos.traverseDown(rootPage, key); + if(!locked && rootReference != getRoot()) { + continue; + } + Page p = pos.page; + int index = pos.index; + tip = pos; + pos = pos.parent; + result = index < 0 ? null : (V)p.getValue(index); + Decision decision = decisionMaker.decide(result, value); + + switch (decision) { + case REPEAT: + decisionMaker.reset(); + continue; + case ABORT: + if(!locked && rootReference != getRoot()) { + decisionMaker.reset(); + continue; + } + return result; + case REMOVE: { + if (index < 0) { + if(!locked && rootReference != getRoot()) { + decisionMaker.reset(); + continue; + } + return null; + } + + if (p.getTotalCount() == 1 && pos != null) { + int keyCount; + do { + p = pos.page; + index = pos.index; + pos = pos.parent; + keyCount = p.getKeyCount(); + // condition below should always be false, but older + // versions (up to 1.4.197) may create + // single-childed (with no keys) internal nodes, + // which we skip here + } while (keyCount == 0 && pos != null); + + if (keyCount <= 1) { + if (keyCount == 1) { + assert index <= 1; + p = p.getChildPage(1 - index); + } else { + // if root happens to be such single-childed + // (with no keys) internal node, then just + // replace it with empty leaf + p = Page.createEmptyLeaf(this); + } + break; + } + } + p = p.copy(); + p.remove(index); + break; + } + case PUT: { + value = decisionMaker.selectValue(result, value); + p = p.copy(); + if (index < 0) { + p.insertLeaf(-index - 1, key, value); + int keyCount; + while ((keyCount = p.getKeyCount()) > store.getKeysPerPage() + || p.getMemory() > store.getMaxPageSize() + && keyCount > (p.isLeaf() ? 1 : 2)) { + long totalCount = p.getTotalCount(); + int at = keyCount >> 1; + Object k = p.getKey(at); + Page split = p.split(at); + unsavedMemoryHolder.value += p.getMemory() + split.getMemory(); + if (pos == null) { + Object[] keys = { k }; + Page.PageReference[] children = { + new Page.PageReference(p), + new Page.PageReference(split) + }; + p = Page.createNode(this, keys, children, totalCount, 0); + break; + } + Page c = p; + p = pos.page; + index = pos.index; + pos = pos.parent; + p = p.copy(); + p.setChild(index, split); + p.insertNode(index, k, c); + } + } else { + p.setValue(index, value); + } + break; + } + } + rootPage = replacePage(pos, p, unsavedMemoryHolder); + if (!locked) { + rootReference = rootReference.updateRootPage(rootPage, attempt); + if (rootReference == null) { + decisionMaker.reset(); + continue; + } + } + store.registerUnsavedMemory(unsavedMemoryHolder.value + tip.processRemovalInfo(version)); + return result; + } finally { + if(locked) { + unlockRoot(rootPage); + } + } + } + } + + private RootReference lockRoot(RootReference rootReference, int attempt) { + while(true) { + RootReference lockedRootReference = tryLock(rootReference, attempt++); + if (lockedRootReference != null) { + return lockedRootReference; + } + rootReference = getRoot(); + } + } + + /** + * Try to lock the root. + * + * @param rootReference the old root reference + * @param attempt the number of attempts so far + * @return the new root reference + */ + protected RootReference tryLock(RootReference rootReference, int attempt) { + RootReference lockedRootReference = rootReference.tryLock(attempt); + if (lockedRootReference != null) { + return lockedRootReference; + } + + RootReference oldRootReference = rootReference.previous; + int contention = 1; + if (oldRootReference != null) { + long updateAttemptCounter = rootReference.updateAttemptCounter - + oldRootReference.updateAttemptCounter; + assert updateAttemptCounter >= 0 : updateAttemptCounter; + long updateCounter = rootReference.updateCounter - oldRootReference.updateCounter; + assert updateCounter >= 0 : updateCounter; + assert updateAttemptCounter >= updateCounter : updateAttemptCounter + " >= " + updateCounter; + contention += (int)((updateAttemptCounter+1) / (updateCounter+1)); + } + + if(attempt > 4) { + if (attempt <= 12) { + Thread.yield(); + } else if (attempt <= 70 - 2 * contention) { + try { + Thread.sleep(contention); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } else { + synchronized (lock) { + notificationRequested = true; + try { + lock.wait(5); + } catch (InterruptedException ignore) { + } + } + } + } + return null; + } + + /** + * Unlock the root page, the new root being null. + * + * @return the new root reference (never null) + */ + private RootReference unlockRoot() { + return unlockRoot(null, -1); + } + + /** + * Unlock the root page. + * + * @param newRootPage the new root + * @return the new root reference (never null) + */ + protected RootReference unlockRoot(Page newRootPage) { + return unlockRoot(newRootPage, -1); + } + + private void unlockRoot(int appendCounter) { + unlockRoot(null, appendCounter); + } + + private RootReference unlockRoot(Page newRootPage, int appendCounter) { + RootReference updatedRootReference; + do { + RootReference rootReference = getRoot(); + assert rootReference.isLockedByCurrentThread(); + updatedRootReference = rootReference.updatePageAndLockedStatus( + newRootPage == null ? rootReference.root : newRootPage, + false, + appendCounter == -1 ? rootReference.getAppendCounter() : appendCounter + ); + } while(updatedRootReference == null); + + notifyWaiters(); + return updatedRootReference; + } + + private void notifyWaiters() { + if (notificationRequested) { + synchronized (lock) { + notificationRequested = false; + lock.notify(); + } + } + } + + private static final class EqualsDecisionMaker extends DecisionMaker { + private final DataType dataType; + private final V expectedValue; + private Decision decision; + + EqualsDecisionMaker(DataType dataType, V expectedValue) { + this.dataType = dataType; + this.expectedValue = expectedValue; + } + + @Override + public Decision decide(V existingValue, V providedValue) { + assert decision == null; + decision = !areValuesEqual(dataType, expectedValue, existingValue) ? Decision.ABORT : + providedValue == null ? Decision.REMOVE : Decision.PUT; + return decision; + } + + @Override + public void reset() { + decision = null; + } + + Decision getDecision() { + return decision; + } + + @Override + public String toString() { + return "equals_to "+expectedValue; + } + } + + private static final class ContainsDecisionMaker extends DecisionMaker { + private Decision decision; + + ContainsDecisionMaker() {} + + @Override + public Decision decide(V existingValue, V providedValue) { + assert decision == null; + decision = existingValue == null ? Decision.ABORT : Decision.PUT; + return decision; + } + + @Override + public T selectValue(T existingValue, T providedValue) { + return existingValue; + } + + @Override + public void reset() { + decision = null; + } + + Decision getDecision() { + return decision; + } + + @Override + public String toString() { + return "contains"; + } + } + + private static final class IntValueHolder { + int value; + + IntValueHolder() {} + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/MVStore.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/MVStore.java new file mode 100644 index 000000000..e76ddf4ee --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/MVStore.java @@ -0,0 +1,3588 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.cache.CacheLongKeyLIRS; +import org.dizitart.no2.mvstore.compat.v1.mvstore.compress.CompressDeflate; +import org.dizitart.no2.mvstore.compat.v1.mvstore.compress.CompressLZF; +import org.dizitart.no2.mvstore.compat.v1.mvstore.compress.Compressor; +import org.h2.util.MathUtils; +import org.h2.util.Utils; + +import java.lang.Thread.UncaughtExceptionHandler; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReentrantLock; + +import static org.dizitart.no2.mvstore.compat.v1.mvstore.MVMap.INITIAL_VERSION; + +/* + +TODO: + +Documentation +- rolling docs review: at "Metadata Map" +- better document that writes are in background thread +- better document how to do non-unique indexes +- document pluggable store and OffHeapStore + +TransactionStore: +- ability to disable the transaction log, + if there is only one connection + +MVStore: +- better and clearer memory usage accounting rules + (heap memory versus disk memory), so that even there is + never an out of memory + even for a small heap, and so that chunks + are still relatively big on average +- make sure serialization / deserialization errors don't corrupt the file +- test and possibly improve compact operation (for large dbs) +- automated 'kill process' and 'power failure' test +- defragment (re-creating maps, specially those with small pages) +- store number of write operations per page (maybe defragment + if much different than count) +- r-tree: nearest neighbor search +- use a small object value cache (StringCache), test on Android + for default serialization +- MVStoreTool.dump should dump the data if possible; + possibly using a callback for serialization +- implement a sharded map (in one store, multiple stores) + to support concurrent updates and writes, and very large maps +- to save space when persisting very small transactions, + use a transaction log where only the deltas are stored +- serialization for lists, sets, sets, sorted sets, maps, sorted maps +- maybe rename 'rollback' to 'revert' to distinguish from transactions +- support other compression algorithms (deflate, LZ4,...) +- remove features that are not really needed; simplify the code + possibly using a separate layer or tools + (retainVersion?) +- optional pluggable checksum mechanism (per page), which + requires that everything is a page (including headers) +- rename "store" to "save", as "store" is used in "storeVersion" +- rename setStoreVersion to setDataVersion, setSchemaVersion or similar +- temporary file storage +- simple rollback method (rollback to last committed version) +- MVMap to implement SortedMap, then NavigableMap +- storage that splits database into multiple files, + to speed up compact and allow using trim + (by truncating / deleting empty files) +- add new feature to the file system API to avoid copying data + (reads that returns a ByteBuffer instead of writing into one) + for memory mapped files and off-heap storage +- support log structured merge style operations (blind writes) + using one map per level plus bloom filter +- have a strict call order MVStore -> MVMap -> Page -> FileStore +- autocommit commits, stores, and compacts from time to time; + the background thread should wait at least 90% of the + configured write delay to store changes +- compact* should also store uncommitted changes (if there are any) +- write a LSM-tree (log structured merge tree) utility on top of the MVStore + with blind writes and/or a bloom filter that + internally uses regular maps and merge sort +- chunk metadata: maybe split into static and variable, + or use a small page size for metadata +- data type "string": maybe use prefix compression for keys +- test chunk id rollover +- feature to auto-compact from time to time and on close +- compact very small chunks +- Page: to save memory, combine keys & values into one array + (also children & counts). Maybe remove some other + fields (childrenCount for example) +- Support SortedMap for MVMap +- compact: copy whole pages (without having to open all maps) +- maybe change the length code to have lower gaps +- test with very low limits (such as: short chunks, small pages) +- maybe allow to read beyond the retention time: + when compacting, move live pages in old chunks + to a map (possibly the metadata map) - + this requires a change in the compaction code, plus + a map lookup when reading old data; also, this + old data map needs to be cleaned up somehow; + maybe using an additional timeout +*/ + +/** + * A persistent storage for maps. + */ +public class MVStore implements AutoCloseable +{ + // The following are attribute names (keys) in store header map + private static final String HDR_H = "H"; + private static final String HDR_BLOCK_SIZE = "blockSize"; + private static final String HDR_FORMAT = "format"; + private static final String HDR_CREATED = "created"; + private static final String HDR_FORMAT_READ = "formatRead"; + private static final String HDR_CHUNK = "chunk"; + private static final String HDR_BLOCK = "block"; + private static final String HDR_VERSION = "version"; + private static final String HDR_CLEAN = "clean"; + private static final String HDR_FLETCHER = "fletcher"; + + + /** + * The block size (physical sector size) of the disk. The store header is + * written twice, one copy in each block, to ensure it survives a crash. + */ + static final int BLOCK_SIZE = 4 * 1024; + + private static final int FORMAT_WRITE = 1; + private static final int FORMAT_READ = 1; + + /** + * Store is open. + */ + private static final int STATE_OPEN = 0; + + /** + * Store is about to close now, but is still operational. + * Outstanding store operation by background writer or other thread may be in progress. + * New updates must not be initiated, unless they are part of a closing procedure itself. + */ + private static final int STATE_STOPPING = 1; + + /** + * Store is closing now, and any operation on it may fail. + */ + private static final int STATE_CLOSING = 2; + + /** + * Store is closed. + */ + private static final int STATE_CLOSED = 3; + + /** + * Lock which governs access to major store operations: store(), close(), ... + * It should used in a non-reentrant fashion. + * It serves as a replacement for synchronized(this), except it allows for + * non-blocking lock attempts. + */ + private final ReentrantLock storeLock = new ReentrantLock(true); + + /** + * Reference to a background thread, which is expected to be running, if any. + */ + private final AtomicReference backgroundWriterThread = new AtomicReference<>(); + + private volatile boolean reuseSpace = true; + + private volatile int state; + + private final FileStore fileStore; + + private final boolean fileStoreIsProvided; + + private final int pageSplitSize; + + private final int keysPerPage; + + /** + * The page cache. The default size is 16 MB, and the average size is 2 KB. + * It is split in 16 segments. The stack move distance is 2% of the expected + * number of entries. + */ + private final CacheLongKeyLIRS cache; + + /** + * The newest chunk. If nothing was stored yet, this field is not set. + */ + private Chunk lastChunk; + + /** + * The map of chunks. + */ + private final ConcurrentHashMap chunks = new ConcurrentHashMap<>(); + + private final Queue removedPages = new PriorityBlockingQueue<>(); + + private final Deque deadChunks = new ArrayDeque<>(); + + private long updateCounter = 0; + private long updateAttemptCounter = 0; + + /** + * The metadata map. Write access to this map needs to be done under storeLock. + */ + private final MVMap meta; + + private final ConcurrentHashMap> maps = new ConcurrentHashMap<>(); + + private final HashMap storeHeader = new HashMap<>(); + + private WriteBuffer writeBuffer; + + private final AtomicInteger lastMapId = new AtomicInteger(); + + private int versionsToKeep = 5; + + /** + * The compression level for new pages (0 for disabled, 1 for fast, 2 for + * high). Even if disabled, the store may contain (old) compressed pages. + */ + private final int compressionLevel; + + private Compressor compressorFast; + + private Compressor compressorHigh; + + private final boolean recoveryMode; + + private final UncaughtExceptionHandler backgroundExceptionHandler; + + private volatile long currentVersion; + + /** + * The version of the last stored chunk, or -1 if nothing was stored so far. + */ + private volatile long lastStoredVersion = INITIAL_VERSION; + + /** + * Oldest store version in use. All version beyond this can be safely dropped + */ + private final AtomicLong oldestVersionToKeep = new AtomicLong(); + + /** + * Ordered collection of all version usage counters for all versions starting + * from oldestVersionToKeep and up to current. + */ + private final Deque versions = new LinkedList<>(); + + /** + * Counter of open transactions for the latest (current) store version + */ + private volatile TxCounter currentTxCounter = new TxCounter(currentVersion); + + /** + * The estimated memory used by unsaved pages. This number is not accurate, + * also because it may be changed concurrently, and because temporary pages + * are counted. + */ + private int unsavedMemory; + private final int autoCommitMemory; + private volatile boolean saveNeeded; + + /** + * The time the store was created, in milliseconds since 1970. + */ + private long creationTime; + + /** + * How long to retain old, persisted chunks, in milliseconds. For larger or + * equal to zero, a chunk is never directly overwritten if unused, but + * instead, the unused field is set. If smaller zero, chunks are directly + * overwritten if unused. + */ + private int retentionTime; + + private long lastCommitTime; + + /** + * The version of the current store operation (if any). + */ + private volatile long currentStoreVersion = -1; + + private volatile boolean metaChanged; + + /** + * The delay in milliseconds to automatically commit and write changes. + */ + private int autoCommitDelay; + + private final int autoCompactFillRate; + private long autoCompactLastFileOpCount; + + private volatile IllegalStateException panicException; + + private long lastTimeAbsolute; + + + /** + * Create and open the store. + * + * @param config the configuration to use + * @throws IllegalStateException if the file is corrupt, or an exception + * occurred while opening + * @throws IllegalArgumentException if the directory does not exist + */ + MVStore(Map config) { + recoveryMode = config.containsKey("recoveryMode"); + compressionLevel = DataUtils.getConfigParam(config, "compress", 0); + String fileName = (String) config.get("fileName"); + FileStore fileStore = (FileStore) config.get("fileStore"); + fileStoreIsProvided = fileStore != null; + if(fileStore == null && fileName != null) { + fileStore = new FileStore(); + } + this.fileStore = fileStore; + + int pgSplitSize = 48; // for "mem:" case it is # of keys + CacheLongKeyLIRS.Config cc = null; + if (this.fileStore != null) { + int mb = DataUtils.getConfigParam(config, "cacheSize", 16); + if (mb > 0) { + cc = new CacheLongKeyLIRS.Config(); + cc.maxMemory = mb * 1024L * 1024L; + Object o = config.get("cacheConcurrency"); + if (o != null) { + cc.segmentCount = (Integer)o; + } + } + pgSplitSize = 16 * 1024; + } + if (cc != null) { + cache = new CacheLongKeyLIRS<>(cc); + } else { + cache = null; + } + + pgSplitSize = DataUtils.getConfigParam(config, "pageSplitSize", pgSplitSize); + // Make sure pages will fit into cache + if (cache != null && pgSplitSize > cache.getMaxItemSize()) { + pgSplitSize = (int)cache.getMaxItemSize(); + } + pageSplitSize = pgSplitSize; + keysPerPage = DataUtils.getConfigParam(config, "keysPerPage", 48); + backgroundExceptionHandler = + (UncaughtExceptionHandler)config.get("backgroundExceptionHandler"); + meta = new MVMap<>(this); + if (this.fileStore != null) { + retentionTime = this.fileStore.getDefaultRetentionTime(); + // 19 KB memory is about 1 KB storage + int kb = Math.max(1, Math.min(19, Utils.scaleForAvailableMemory(64))) * 1024; + kb = DataUtils.getConfigParam(config, "autoCommitBufferSize", kb); + autoCommitMemory = kb * 1024; + autoCompactFillRate = DataUtils.getConfigParam(config, "autoCompactFillRate", 90); + char[] encryptionKey = (char[]) config.get("encryptionKey"); + try { + if (!fileStoreIsProvided) { + boolean readOnly = config.containsKey("readOnly"); + this.fileStore.open(fileName, readOnly, encryptionKey); + } + if (this.fileStore.size() == 0) { + creationTime = getTimeAbsolute(); + lastCommitTime = creationTime; + storeHeader.put(HDR_H, 2); + storeHeader.put(HDR_BLOCK_SIZE, BLOCK_SIZE); + storeHeader.put(HDR_FORMAT, FORMAT_WRITE); + storeHeader.put(HDR_CREATED, creationTime); + writeStoreHeader(); + } else { + // there is no need to lock store here, since it is not opened yet, + // just to make some assertions happy, when they ensure single-threaded access + storeLock.lock(); + try { + readStoreHeader(); + } finally { + storeLock.unlock(); + } + } + } catch (IllegalStateException e) { + panic(e); + } finally { + if (encryptionKey != null) { + Arrays.fill(encryptionKey, (char) 0); + } + } + lastCommitTime = getTimeSinceCreation(); + + scrubMetaMap(); + + // setAutoCommitDelay starts the thread, but only if + // the parameter is different from the old value + int delay = DataUtils.getConfigParam(config, "autoCommitDelay", 1000); + setAutoCommitDelay(delay); + } else { + autoCommitMemory = 0; + autoCompactFillRate = 0; + } + } + + private void scrubMetaMap() { + Set keysToRemove = new HashSet<>(); + + // ensure that there is only one name mapped to this id + // this could be a leftover of an unfinished map rename + for (Iterator it = meta.keyIterator(DataUtils.META_NAME); it.hasNext();) { + String key = it.next(); + if (!key.startsWith(DataUtils.META_NAME)) { + break; + } + String mapName = key.substring(DataUtils.META_NAME.length()); + int mapId = DataUtils.parseHexInt(meta.get(key)); + String realMapName = getMapName(mapId); + if(!mapName.equals(realMapName)) { + keysToRemove.add(key); + } + } + + // remove roots of non-existent maps (leftover after unfinished map removal) + for (Iterator it = meta.keyIterator(DataUtils.META_ROOT); it.hasNext();) { + String key = it.next(); + if (!key.startsWith(DataUtils.META_ROOT)) { + break; + } + String mapIdStr = key.substring(key.lastIndexOf('.') + 1); + if(!meta.containsKey(DataUtils.META_MAP + mapIdStr)) { + meta.remove(key); + markMetaChanged(); + keysToRemove.add(key); + } + } + + for (String key : keysToRemove) { + meta.remove(key); + markMetaChanged(); + } + + for (Iterator it = meta.keyIterator(DataUtils.META_MAP); it.hasNext();) { + String key = it.next(); + if (!key.startsWith(DataUtils.META_MAP)) { + break; + } + String mapName = DataUtils.getMapName(meta.get(key)); + String mapIdStr = key.substring(DataUtils.META_MAP.length()); + // ensure that last map id is not smaller than max of any existing map ids + int mapId = DataUtils.parseHexInt(mapIdStr); + if (mapId > lastMapId.get()) { + lastMapId.set(mapId); + } + // each map should have a proper name + if(!mapIdStr.equals(meta.get(DataUtils.META_NAME + mapName))) { + meta.put(DataUtils.META_NAME + mapName, mapIdStr); + markMetaChanged(); + } + } + } + + private void panic(IllegalStateException e) { + if (isOpen()) { + handleException(e); + panicException = e; + closeImmediately(); + } + throw e; + } + + public IllegalStateException getPanicException() { + return panicException; + } + + /** + * Open a store in exclusive mode. For a file-based store, the parent + * directory must already exist. + * + * @param fileName the file name (null for in-memory) + * @return the store + */ + public static MVStore open(String fileName) { + HashMap config = new HashMap<>(); + config.put("fileName", fileName); + return new MVStore(config); + } + + /** + * Open a map with the default settings. The map is automatically create if + * it does not yet exist. If a map with this name is already open, this map + * is returned. + * + * @param the key type + * @param the value type + * @param name the name of the map + * @return the map + */ + public MVMap openMap(String name) { + return openMap(name, new MVMap.Builder()); + } + + /** + * Open a map with the given builder. The map is automatically create if it + * does not yet exist. If a map with this name is already open, this map is + * returned. + * + * @param the map type + * @param the key type + * @param the value type + * @param name the name of the map + * @param builder the map builder + * @return the map + */ + public , K, V> M openMap(String name, MVMap.MapBuilder builder) { + int id = getMapId(name); + M map; + if (id >= 0) { + map = openMap(id, builder); + assert builder.getKeyType() == null || map.getKeyType().getClass().equals(builder.getKeyType().getClass()); + assert builder.getValueType() == null || map.getValueType().getClass().equals(builder.getValueType() + .getClass()); + } else { + HashMap c = new HashMap<>(); + id = lastMapId.incrementAndGet(); + assert getMap(id) == null; + c.put("id", id); + c.put("createVersion", currentVersion); + map = builder.create(this, c); + String x = Integer.toHexString(id); + meta.put(MVMap.getMapKey(id), map.asString(name)); + meta.put(DataUtils.META_NAME + name, x); + map.setRootPos(0, lastStoredVersion); + markMetaChanged(); + @SuppressWarnings("unchecked") + M existingMap = (M) maps.putIfAbsent(id, map); + if (existingMap != null) { + map = existingMap; + } + } + return map; + } + + private , K, V> M openMap(int id, MVMap.MapBuilder builder) { + storeLock.lock(); + try { + @SuppressWarnings("unchecked") + M map = (M) getMap(id); + if (map == null) { + String configAsString = meta.get(MVMap.getMapKey(id)); + HashMap config; + if (configAsString != null) { + config = new HashMap(DataUtils.parseMap(configAsString)); + } else { + config = new HashMap<>(); + } + config.put("id", id); + map = builder.create(this, config); + long root = getRootPos(meta, id); + map.setRootPos(root, lastStoredVersion); + maps.put(id, map); + } + return map; + } finally { + storeLock.unlock(); + } + } + + /** + * Get map by id. + * + * @param the key type + * @param the value type + * @param id map id + * @return Map + */ + public MVMap getMap(int id) { + checkOpen(); + @SuppressWarnings("unchecked") + MVMap map = (MVMap) maps.get(id); + return map; + } + + /** + * Get the set of all map names. + * + * @return the set of names + */ + public Set getMapNames() { + HashSet set = new HashSet<>(); + checkOpen(); + for (Iterator it = meta.keyIterator(DataUtils.META_NAME); it.hasNext();) { + String x = it.next(); + if (!x.startsWith(DataUtils.META_NAME)) { + break; + } + String mapName = x.substring(DataUtils.META_NAME.length()); + set.add(mapName); + } + return set; + } + + /** + * Get the metadata map. This data is for informational purposes only. The + * data is subject to change in future versions. + *

+ * The data in this map should not be modified (changing system data may + * corrupt the store). If modifications are needed, they need be + * synchronized on the store. + *

+ * The metadata map contains the following entries: + * 

+     * chunk.{chunkId} = {chunk metadata}
+     * name.{name} = {mapId}
+     * map.{mapId} = {map metadata}
+     * root.{mapId} = {root position}
+     * setting.storeVersion = {version}
+     *
+ * + * @return the metadata map + */ + public MVMap getMetaMap() { + checkOpen(); + return meta; + } + + private MVMap getMetaMap(long version) { + Chunk c = getChunkForVersion(version); + DataUtils.checkArgument(c != null, "Unknown version {0}", version); + long block = c.block; + c = readChunkHeader(block); + MVMap oldMeta = meta.openReadOnly(c.metaRootPos, version); + return oldMeta; + } + + private Chunk getChunkForVersion(long version) { + Chunk newest = null; + for (Chunk c : chunks.values()) { + if (c.version <= version) { + if (newest == null || c.id > newest.id) { + newest = c; + } + } + } + return newest; + } + + /** + * Check whether a given map exists. + * + * @param name the map name + * @return true if it exists + */ + public boolean hasMap(String name) { + return meta.containsKey(DataUtils.META_NAME + name); + } + + /** + * Check whether a given map exists and has data. + * + * @param name the map name + * @return true if it exists and has data. + */ + public boolean hasData(String name) { + return hasMap(name) && getRootPos(meta, getMapId(name)) != 0; + } + + private void markMetaChanged() { + // changes in the metadata alone are usually not detected, as the meta + // map is changed after storing + metaChanged = true; + } + + private void readStoreHeader() { + Chunk newest = null; + boolean assumeCleanShutdown = true; + boolean validStoreHeader = false; + // find out which chunk and version are the newest + // read the first two blocks + ByteBuffer fileHeaderBlocks = fileStore.readFully(0, 2 * BLOCK_SIZE); + byte[] buff = new byte[BLOCK_SIZE]; + for (int i = 0; i <= BLOCK_SIZE; i += BLOCK_SIZE) { + fileHeaderBlocks.get(buff); + // the following can fail for various reasons + try { + HashMap m = DataUtils.parseChecksummedMap(buff); + if (m == null) { + assumeCleanShutdown = false; + continue; + } + long version = DataUtils.readHexLong(m, HDR_VERSION, 0); + // if both header blocks do agree on version + // we'll continue on happy path - assume that previous shutdown was clean + assumeCleanShutdown = assumeCleanShutdown && (newest == null || version == newest.version); + if (newest == null || version > newest.version) { + validStoreHeader = true; + storeHeader.putAll(m); + creationTime = DataUtils.readHexLong(m, HDR_CREATED, 0); + int chunkId = DataUtils.readHexInt(m, HDR_CHUNK, 0); + long block = DataUtils.readHexLong(m, HDR_BLOCK, 0); + Chunk test = readChunkHeaderAndFooter(block, chunkId); + if (test != null) { + newest = test; + } + } + } catch (Exception ignore) { + assumeCleanShutdown = false; + } + } + + if (!validStoreHeader) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_CORRUPT, + "Store header is corrupt: {0}", fileStore); + } + int blockSize = DataUtils.readHexInt(storeHeader, HDR_BLOCK_SIZE, BLOCK_SIZE); + if (blockSize != BLOCK_SIZE) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_UNSUPPORTED_FORMAT, + "Block size {0} is currently not supported", + blockSize); + } + long format = DataUtils.readHexLong(storeHeader, HDR_FORMAT, 1); + if (format > FORMAT_WRITE && !fileStore.isReadOnly()) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_UNSUPPORTED_FORMAT, + "The write format {0} is larger " + + "than the supported format {1}, " + + "and the file was not opened in read-only mode", + format, FORMAT_WRITE); + } + format = DataUtils.readHexLong(storeHeader, HDR_FORMAT_READ, format); + if (format > FORMAT_READ) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_UNSUPPORTED_FORMAT, + "The read format {0} is larger " + + "than the supported format {1}", + format, FORMAT_READ); + } + + assumeCleanShutdown = assumeCleanShutdown && newest != null + && DataUtils.readHexInt(storeHeader, HDR_CLEAN, 0) != 0 + && !recoveryMode; + lastStoredVersion = INITIAL_VERSION; + chunks.clear(); + long now = System.currentTimeMillis(); + // calculate the year (doesn't have to be exact; + // we assume 365.25 days per year, * 4 = 1461) + int year = 1970 + (int) (now / (1000L * 60 * 60 * 6 * 1461)); + if (year < 2014) { + // if the year is before 2014, + // we assume the system doesn't have a real-time clock, + // and we set the creationTime to the past, so that + // existing chunks are overwritten + creationTime = now - fileStore.getDefaultRetentionTime(); + } else if (now < creationTime) { + // the system time was set to the past: + // we change the creation time + creationTime = now; + storeHeader.put(HDR_CREATED, creationTime); + } + + long fileSize = fileStore.size(); + long blocksInStore = fileSize / BLOCK_SIZE; + + Comparator chunkComparator = new Comparator() { + @Override + public int compare(Chunk one, Chunk two) { + int result = Long.compare(two.version, one.version); + if (result == 0) { + // out of two copies of the same chunk we prefer the one + // close to the beginning of file (presumably later version) + result = Long.compare(one.block, two.block); + } + return result; + } + }; + + if (!assumeCleanShutdown) { + Chunk tailChunk = discoverChunk(blocksInStore); + if (tailChunk != null) { + blocksInStore = tailChunk.block; // for a possible full scan later on + if (newest == null || tailChunk.version > newest.version) { + newest = tailChunk; + } + } + } + + Map validChunksByLocation = new HashMap<>(); + if (newest != null) { + // read the chunk header and footer, + // and follow the chain of next chunks + while (true) { + validChunksByLocation.put(newest.block, newest); + if (newest.next == 0 || newest.next >= blocksInStore) { + // no (valid) next + break; + } + Chunk test = readChunkHeaderAndFooter(newest.next, newest.id + 1); + if (test == null || test.version <= newest.version) { + break; + } + // if shutdown was really clean then chain should be empty + assumeCleanShutdown = false; + newest = test; + } + } + + if (assumeCleanShutdown) { + setLastChunk(newest); + // quickly check latest 20 chunks referenced in meta table + Queue chunksToVerify = new PriorityQueue<>(20, Collections.reverseOrder(chunkComparator)); + try { + // load the chunk metadata: although meta's root page resides in the lastChunk, + // traversing meta map might recursively load another chunk(s) + Cursor cursor = meta.cursor(DataUtils.META_CHUNK); + while (cursor.hasNext() && cursor.next().startsWith(DataUtils.META_CHUNK)) { + Chunk c = Chunk.fromString(cursor.getValue()); + assert c.version <= currentVersion; + // might be there already, due to meta traversal + // see readPage() ... getChunkIfFound() + chunks.putIfAbsent(c.id, c); + chunksToVerify.offer(c); + if (chunksToVerify.size() == 20) { + chunksToVerify.poll(); + } + } + Chunk c; + while (assumeCleanShutdown && (c = chunksToVerify.poll()) != null) { + assumeCleanShutdown = readChunkHeaderAndFooter(c.block, c.id) != null; + } + } catch(IllegalStateException ignored) { + assumeCleanShutdown = false; + } + } + + if (!assumeCleanShutdown) { + boolean quickRecovery = false; + if (!recoveryMode) { + // now we know, that previous shutdown did not go well and file + // is possibly corrupted but there is still hope for a quick + // recovery + + // this collection will hold potential candidates for lastChunk to fall back to, + // in order from the most to least likely + Chunk[] lastChunkCandidates = validChunksByLocation.values().toArray(new Chunk[0]); + Arrays.sort(lastChunkCandidates, chunkComparator); + Map validChunksById = new HashMap<>(); + for (Chunk chunk : lastChunkCandidates) { + validChunksById.put(chunk.id, chunk); + } + quickRecovery = findLastChunkWithCompleteValidChunkSet(lastChunkCandidates, validChunksByLocation, + validChunksById, false); + } + + if (!quickRecovery) { + // scan whole file and try to fetch chunk header and/or footer out of every block + // matching pairs with nothing in-between are considered as valid chunk + long block = blocksInStore; + Chunk tailChunk; + while ((tailChunk = discoverChunk(block)) != null) { + block = tailChunk.block; + validChunksByLocation.put(block, tailChunk); + } + + // this collection will hold potential candidates for lastChunk to fall back to, + // in order from the most to least likely + Chunk[] lastChunkCandidates = validChunksByLocation.values().toArray(new Chunk[0]); + Arrays.sort(lastChunkCandidates, chunkComparator); + Map validChunksById = new HashMap<>(); + for (Chunk chunk : lastChunkCandidates) { + validChunksById.put(chunk.id, chunk); + } + findLastChunkWithCompleteValidChunkSet(lastChunkCandidates, validChunksByLocation, + validChunksById, true); + } + } + + fileStore.clear(); + // build the free space list + for (Chunk c : chunks.values()) { + if (c.isSaved()) { + long start = c.block * BLOCK_SIZE; + int length = c.len * BLOCK_SIZE; + fileStore.markUsed(start, length); + } + if (!c.isLive()) { + deadChunks.offer(c); + } + } + assert validateFileLength("on open"); + setWriteVersion(currentVersion); + if (lastStoredVersion == INITIAL_VERSION) { + lastStoredVersion = currentVersion - 1; + } + } + + private boolean findLastChunkWithCompleteValidChunkSet(Chunk[] lastChunkCandidates, + Map validChunksByLocation, + Map validChunksById, + boolean afterFullScan) { + // Try candidates for "last chunk" in order from newest to oldest + // until suitable is found. Suitable one should have meta map + // where all chunk references point to valid locations. + for (Chunk chunk : lastChunkCandidates) { + boolean verified = true; + try { + setLastChunk(chunk); + // load the chunk metadata: although meta's root page resides in the lastChunk, + // traversing meta map might recursively load another chunk(s) + Cursor cursor = meta.cursor(DataUtils.META_CHUNK); + while (cursor.hasNext() && cursor.next().startsWith(DataUtils.META_CHUNK)) { + Chunk c = Chunk.fromString(cursor.getValue()); + assert c.version <= currentVersion; + // might be there already, due to meta traversal + // see readPage() ... getChunkIfFound() + Chunk test = chunks.putIfAbsent(c.id, c); + if (test != null) { + c = test; + } + assert chunks.get(c.id) == c; + if ((test = validChunksByLocation.get(c.block)) == null || test.id != c.id) { + if ((test = validChunksById.get(c.id)) != null) { + // We do not have a valid chunk at that location, + // but there is a copy of same chunk from original + // location. + // Chunk header at original location does not have + // any dynamic (occupancy) metadata, so it can't be + // used here as is, re-point our chunk to original + // location instead. + c.block = test.block; + } else if (!c.isLive()) { + // we can just remove entry from meta, referencing to this chunk, + // but store maybe R/O, and it's not properly started yet, + // so lets make this chunk "dead" and taking no space, + // and it will be automatically removed later. + c.block = Long.MAX_VALUE; + c.len = Integer.MAX_VALUE; + if (c.unused == 0) { + c.unused = creationTime; + } + if (c.unusedAtVersion == 0) { + c.unusedAtVersion = INITIAL_VERSION; + } + } else if (afterFullScan || readChunkHeaderAndFooter(c.block, c.id) == null) { + // chunk reference is invalid + // this "last chunk" candidate is not suitable + verified = false; + break; + } + } + } + } catch(Exception ignored) { + verified = false; + } + if (verified) { + return true; + } + } + return false; + } + + private void setLastChunk(Chunk last) { + chunks.clear(); + lastChunk = last; + if (last == null) { + // no valid chunk + lastMapId.set(0); + currentVersion = 0; + lastStoredVersion = INITIAL_VERSION; + meta.setRootPos(0, INITIAL_VERSION); + } else { + lastMapId.set(last.mapId); + currentVersion = last.version; + chunks.put(last.id, last); + lastStoredVersion = currentVersion - 1; + meta.setRootPos(last.metaRootPos, lastStoredVersion); + } + } + + /** + * Discover a valid chunk, searching file backwards from the given block + * + * @param block to start search from (found chunk footer should be no + * further than block-1) + * @return valid chunk or null if none found + */ + private Chunk discoverChunk(long block) { + long candidateLocation = Long.MAX_VALUE; + Chunk candidate = null; + while (true) { + if (block == candidateLocation) { + return candidate; + } + if (block == 2) { // number of blocks occupied by headers + return null; + } + Chunk test = readChunkFooter(block); + if (test != null) { + // if we encounter chunk footer (with or without corresponding header) + // in the middle of prospective chunk, stop considering it + candidateLocation = Long.MAX_VALUE; + test = readChunkHeaderOptionally(test.block, test.id); + if (test != null) { + // if that footer has a corresponding header, + // consider them as a new candidate for a valid chunk + candidate = test; + candidateLocation = test.block; + } + } + + // if we encounter chunk header without corresponding footer + // (due to incomplete write?) in the middle of prospective + // chunk, stop considering it + if (--block > candidateLocation && readChunkHeaderOptionally(block) != null) { + candidateLocation = Long.MAX_VALUE; + } + } + } + + + /** + * Read a chunk header and footer, and verify the stored data is consistent. + * + * @param block the block + * @param expectedId of the chunk + * @return the chunk, or null if the header or footer don't match or are not + * consistent + */ + private Chunk readChunkHeaderAndFooter(long block, int expectedId) { + Chunk header = readChunkHeaderOptionally(block, expectedId); + if (header != null) { + Chunk footer = readChunkFooter(block + header.len); + if (footer == null || footer.id != expectedId || footer.block != header.block) { + return null; + } + } + return header; + } + + /** + * Try to read a chunk footer. + * + * @param block the index of the next block after the chunk + * @return the chunk, or null if not successful + */ + private Chunk readChunkFooter(long block) { + // the following can fail for various reasons + try { + // read the chunk footer of the last block of the file + long pos = block * BLOCK_SIZE - Chunk.FOOTER_LENGTH; + if(pos < 0) { + return null; + } + ByteBuffer lastBlock = fileStore.readFully(pos, Chunk.FOOTER_LENGTH); + byte[] buff = new byte[Chunk.FOOTER_LENGTH]; + lastBlock.get(buff); + HashMap m = DataUtils.parseChecksummedMap(buff); + if (m != null) { + int chunk = DataUtils.readHexInt(m, HDR_CHUNK, 0); + Chunk c = new Chunk(chunk); + c.version = DataUtils.readHexLong(m, HDR_VERSION, 0); + c.block = DataUtils.readHexLong(m, HDR_BLOCK, 0); + return c; + } + } catch (Exception e) { + // ignore + } + return null; + } + + private void writeStoreHeader() { + StringBuilder buff = new StringBuilder(112); + if (lastChunk != null) { + storeHeader.put(HDR_BLOCK, lastChunk.block); + storeHeader.put(HDR_CHUNK, lastChunk.id); + storeHeader.put(HDR_VERSION, lastChunk.version); + } + DataUtils.appendMap(buff, storeHeader); + byte[] bytes = buff.toString().getBytes(StandardCharsets.ISO_8859_1); + int checksum = DataUtils.getFletcher32(bytes, 0, bytes.length); + DataUtils.appendMap(buff, HDR_FLETCHER, checksum); + buff.append('\n'); + bytes = buff.toString().getBytes(StandardCharsets.ISO_8859_1); + ByteBuffer header = ByteBuffer.allocate(2 * BLOCK_SIZE); + header.put(bytes); + header.position(BLOCK_SIZE); + header.put(bytes); + header.rewind(); + write(0, header); + } + + private void write(long pos, ByteBuffer buffer) { + try { + fileStore.writeFully(pos, buffer); + } catch (IllegalStateException e) { + panic(e); + } + } + + /** + * Close the file and the store. Unsaved changes are written to disk first. + */ + @Override + public void close() { + closeStore(true, 0); + } + + /** + * Close the file and the store. Unsaved changes are written to disk first, + * and compaction (up to a specified number of milliseconds) is attempted. + * + * @param allowedCompactionTime the allowed time for compaction (in + * milliseconds) + */ + public void close(long allowedCompactionTime) { + closeStore(true, allowedCompactionTime); + } + + /** + * Close the file and the store, without writing anything. + * This will try to stop the background thread (without waiting for it). + * This method ignores all errors. + */ + public void closeImmediately() { + try { + closeStore(false, 0); + } catch (Throwable e) { + handleException(e); + } + } + + private void closeStore(boolean normalShutdown, long allowedCompactionTime) { + // If any other thead have already initiated closure procedure, + // isClosed() would wait until closure is done and then we jump out of the loop. + // This is a subtle difference between !isClosed() and isOpen(). + while (!isClosed()) { + stopBackgroundThread(normalShutdown); + storeLock.lock(); + try { + if (state == STATE_OPEN) { + state = STATE_STOPPING; + try { + try { + if (normalShutdown && fileStore != null && !fileStore.isReadOnly()) { + for (MVMap map : maps.values()) { + if (map.isClosed()) { + deregisterMapRoot(map.getId()); + } + } + setRetentionTime(0); + commit(); + if (allowedCompactionTime > 0) { + compactFile(allowedCompactionTime); + } else if (allowedCompactionTime < 0) { + doMaintenance(autoCompactFillRate); + } + shrinkFileIfPossible(0); + storeHeader.put(HDR_CLEAN, 1); + writeStoreHeader(); + sync(); + assert validateFileLength("on close"); + } + + state = STATE_CLOSING; + + // release memory early - this is important when called + // because of out of memory + clearCaches(); + for (MVMap m : new ArrayList<>(maps.values())) { + m.close(); + } + chunks.clear(); + maps.clear(); + } finally { + if (fileStore != null && !fileStoreIsProvided) { + fileStore.close(); + } + } + } finally { + state = STATE_CLOSED; + } + } + } finally { + storeLock.unlock(); + } + } + } + + /** + * Read a page of data into a ByteBuffer. + * + * @param pos page pos + * @param expectedMapId expected map id for the page + * @return ByteBuffer containing page data. + */ + private ByteBuffer readBufferForPage(long pos, int expectedMapId) { + return getChunk(pos).readBufferForPage(fileStore, pos, expectedMapId); + } + + /** + * Get the chunk for the given position. + * + * @param pos the position + * @return the chunk + */ + private Chunk getChunk(long pos) { + int chunkId = DataUtils.getPageChunkId(pos); + Chunk c = chunks.get(chunkId); + if (c == null) { + checkOpen(); + String s = meta.get(Chunk.getMetaKey(chunkId)); + if (s == null) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_CHUNK_NOT_FOUND, + "Chunk {0} not found", chunkId); + } + c = Chunk.fromString(s); + if (!c.isSaved()) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_CORRUPT, + "Chunk {0} is invalid", chunkId); + } + chunks.put(c.id, c); + } + return c; + } + + private void setWriteVersion(long version) { + for (Iterator> iter = maps.values().iterator(); iter.hasNext(); ) { + MVMap map = iter.next(); + assert map != meta; + if (map.setWriteVersion(version) == null) { + iter.remove(); + } + } + meta.setWriteVersion(version); + onVersionChange(version); + } + + /** + * Unlike regular commit this method returns immediately if there is commit + * in progress on another thread, otherwise it acts as regular commit. + * + * This method may return BEFORE this thread changes are actually persisted! + * + * @return the new version (incremented if there were changes) + */ + public long tryCommit() { + // we need to prevent re-entrance, which may be possible, + // because meta map is modified within storeNow() and that + // causes beforeWrite() call with possibility of going back here + if ((!storeLock.isHeldByCurrentThread() || currentStoreVersion < 0) && + storeLock.tryLock()) { + try { + store(); + } finally { + storeLock.unlock(); + } + } + return currentVersion; + } + + /** + * Commit the changes. + *

+ * This method does nothing if there are no unsaved changes, + * otherwise it increments the current version + * and stores the data (for file based stores). + *

+ * It is not necessary to call this method when auto-commit is enabled (the default + * setting), as in this case it is automatically called from time to time or + * when enough changes have accumulated. However, it may still be called to + * flush all changes to disk. + *

+ * At most one store operation may run at any time. + * + * @return the new version (incremented if there were changes) + */ + public long commit() { + // we need to prevent re-entrance, which may be possible, + // because meta map is modified within storeNow() and that + // causes beforeWrite() call with possibility of going back here + if(!storeLock.isHeldByCurrentThread() || currentStoreVersion < 0) { + storeLock.lock(); + try { + store(); + } finally { + storeLock.unlock(); + } + } + return currentVersion; + } + + private void store() { + store(0, reuseSpace ? 0 : getAfterLastBlock()); + } + + private void store(long reservedLow, long reservedHigh) { + assert storeLock.isHeldByCurrentThread(); + if (isOpenOrStopping()) { + if (hasUnsavedChanges()) { + dropUnusedChunks(); + try { + currentStoreVersion = currentVersion; + if (fileStore == null) { + lastStoredVersion = currentVersion; + //noinspection NonAtomicOperationOnVolatileField + ++currentVersion; + setWriteVersion(currentVersion); + metaChanged = false; + } else { + if (fileStore.isReadOnly()) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_WRITING_FAILED, "This store is read-only"); + } + try { + storeNow(reservedLow, reservedHigh); + } catch (IllegalStateException e) { + panic(e); + } catch (Throwable e) { + panic(DataUtils.newIllegalStateException(DataUtils.ERROR_INTERNAL, "{0}", e.toString(), + e)); + } + } + } finally { + // in any case reset the current store version, + // to allow closing the store + currentStoreVersion = -1; + } + } + } + } + + private void storeNow(long reservedLow, long reservedHigh) { + long time = getTimeSinceCreation(); + int currentUnsavedPageCount = unsavedMemory; + long storeVersion = currentStoreVersion; + long version = ++currentVersion; + lastCommitTime = time; + + // the metadata of the last chunk was not stored so far, and needs to be + // set now (it's better not to update right after storing, because that + // would modify the meta map again) + int lastChunkId; + if (lastChunk == null) { + lastChunkId = 0; + } else { + lastChunkId = lastChunk.id; + meta.put(Chunk.getMetaKey(lastChunkId), lastChunk.asString()); + markMetaChanged(); + // never go backward in time + time = Math.max(lastChunk.time, time); + } + int newChunkId = lastChunkId; + while (true) { + newChunkId = (newChunkId + 1) & Chunk.MAX_ID; + Chunk old = chunks.get(newChunkId); + if (old == null) { + break; + } + if (!old.isSaved()) { + IllegalStateException e = DataUtils.newIllegalStateException( + DataUtils.ERROR_INTERNAL, + "Last block {0} not stored, possibly due to out-of-memory", old); + panic(e); + } + } + Chunk c = new Chunk(newChunkId); + c.pageCount = 0; + c.pageCountLive = 0; + c.maxLen = 0; + c.maxLenLive = 0; + c.metaRootPos = Long.MAX_VALUE; + c.block = Long.MAX_VALUE; + c.len = Integer.MAX_VALUE; + c.time = time; + c.version = version; + c.next = Long.MAX_VALUE; + chunks.put(c.id, c); + ArrayList changed = new ArrayList<>(); + for (Iterator> iter = maps.values().iterator(); iter.hasNext(); ) { + MVMap map = iter.next(); + RootReference rootReference = map.setWriteVersion(version); + if (rootReference == null) { + iter.remove(); + } else if (map.getCreateVersion() <= storeVersion && // if map was created after storing started, skip it + !map.isVolatile() && + map.hasChangesSince(lastStoredVersion)) { + assert rootReference.version <= version : rootReference.version + " > " + version; + Page rootPage = rootReference.root; + if (!rootPage.isSaved() || + // after deletion previously saved leaf + // may pop up as a root, but we still need + // to save new root pos in meta + rootPage.isLeaf()) { + changed.add(rootPage); + } + } + } + WriteBuffer buff = getWriteBuffer(); + // need to patch the header later + c.writeChunkHeader(buff, 0); + int headerLength = buff.position() + 44; + buff.position(headerLength); + for (Page p : changed) { + String key = MVMap.getMapRootKey(p.getMapId()); + if (p.getTotalCount() == 0) { + meta.remove(key); + } else { + p.writeUnsavedRecursive(c, buff); + long root = p.getPos(); + meta.put(key, Long.toHexString(root)); + } + } + + acceptChunkOccupancyChanges(time, version); + + RootReference metaRootReference = meta.setWriteVersion(version); + assert metaRootReference != null; + assert metaRootReference.version == version : metaRootReference.version + " != " + version; + metaChanged = false; + + acceptChunkOccupancyChanges(time, version); + + onVersionChange(version); + + Page metaRoot = metaRootReference.root; + metaRoot.writeUnsavedRecursive(c, buff); + + // last allocated map id should be captured after the meta map was saved, because + // this will ensure that concurrently created map, which made it into meta before save, + // will have it's id reflected in mapid field of currently written chunk + c.mapId = lastMapId.get(); + + int chunkLength = buff.position(); + + // add the store header and round to the next block + int length = MathUtils.roundUpInt(chunkLength + + Chunk.FOOTER_LENGTH, BLOCK_SIZE); + buff.limit(length); + + long filePos = fileStore.allocate(length, reservedLow, reservedHigh); + c.block = filePos / BLOCK_SIZE; + c.len = length / BLOCK_SIZE; + assert validateFileLength(c.asString()); + c.metaRootPos = metaRoot.getPos(); + // calculate and set the likely next position + if (reservedLow > 0 || reservedHigh == reservedLow) { + c.next = fileStore.predictAllocation(c.len, 0, 0); + } else { + // just after this chunk + c.next = 0; + } + assert c.pageCountLive == c.pageCount : c; + buff.position(0); + c.writeChunkHeader(buff, headerLength); + + buff.position(buff.limit() - Chunk.FOOTER_LENGTH); + buff.put(c.getFooterBytes()); + + buff.position(0); + write(filePos, buff.getBuffer()); + releaseWriteBuffer(buff); + + // whether we need to write the store header + boolean writeStoreHeader = false; + // end of the used space is not necessarily the end of the file + boolean storeAtEndOfFile = filePos + length >= fileStore.size(); + if (!storeAtEndOfFile) { + if (lastChunk == null) { + writeStoreHeader = true; + } else if (lastChunk.next != c.block) { + // the last prediction did not matched + writeStoreHeader = true; + } else { + long headerVersion = DataUtils.readHexLong(storeHeader, HDR_VERSION, 0); + if (lastChunk.version - headerVersion > 20) { + // we write after at least every 20 versions + writeStoreHeader = true; + } else { + int chunkId = DataUtils.readHexInt(storeHeader, HDR_CHUNK, 0); + while (true) { + Chunk old = chunks.get(chunkId); + if (old == null) { + // one of the chunks in between + // was removed + writeStoreHeader = true; + break; + } + if (chunkId == lastChunk.id) { + break; + } + chunkId++; + } + } + } + } + + if (storeHeader.remove(HDR_CLEAN) != null) { + writeStoreHeader = true; + } + + lastChunk = c; + if (writeStoreHeader) { + writeStoreHeader(); + } + if (!storeAtEndOfFile) { + // may only shrink after the store header was written + shrinkFileIfPossible(1); + } + for (Page p : changed) { + p.writeEnd(); + } + metaRoot.writeEnd(); + + // some pages might have been changed in the meantime (in the newest + // version) + saveNeeded = false; + unsavedMemory = Math.max(0, unsavedMemory - currentUnsavedPageCount); + lastStoredVersion = storeVersion; + } + + /** + * Get a buffer for writing. This caller must synchronize on the store + * before calling the method and until after using the buffer. + * + * @return the buffer + */ + private WriteBuffer getWriteBuffer() { + WriteBuffer buff; + if (writeBuffer != null) { + buff = writeBuffer; + buff.clear(); + } else { + buff = new WriteBuffer(); + } + return buff; + } + + /** + * Release a buffer for writing. This caller must synchronize on the store + * before calling the method and until after using the buffer. + * + * @param buff the buffer than can be re-used + */ + private void releaseWriteBuffer(WriteBuffer buff) { + if (buff.capacity() <= 4 * 1024 * 1024) { + writeBuffer = buff; + } + } + + private static boolean canOverwriteChunk(Chunk c, long oldestVersionToKeep) { + return !c.isLive() && c.unusedAtVersion < oldestVersionToKeep; + } + + private boolean isSeasonedChunk(Chunk chunk, long time) { + return retentionTime < 0 || chunk.time + retentionTime <= time; + } + + private long getTimeSinceCreation() { + return Math.max(0, getTimeAbsolute() - creationTime); + } + + private long getTimeAbsolute() { + long now = System.currentTimeMillis(); + if (lastTimeAbsolute != 0 && now < lastTimeAbsolute) { + // time seems to have run backwards - this can happen + // when the system time is adjusted, for example + // on a leap second + now = lastTimeAbsolute; + } else { + lastTimeAbsolute = now; + } + return now; + } + + /** + * Apply the freed space to the chunk metadata. The metadata is updated, but + * completely free chunks are not removed from the set of chunks, and the + * disk space is not yet marked as free. They are queued instead and wait until + * their usage is over. + */ + private void acceptChunkOccupancyChanges(long time, long version) { + Set modifiedChunks = new HashSet<>(); + while (true) { + RemovedPageInfo rpi; + while ((rpi = removedPages.peek()) != null && rpi.version < version) { + rpi = removedPages.poll(); // could be different from the peeked one + assert rpi != null; // since nobody else retrieves from queue + assert rpi.version < version : rpi + " < " + version; + int chunkId = rpi.getPageChunkId(); + Chunk chunk = chunks.get(chunkId); + assert chunk != null; + modifiedChunks.add(chunk); + if (chunk.accountForRemovedPage(rpi.getPageLength(), rpi.isPinned(), time, rpi.version)) { + deadChunks.offer(chunk); + } + } + if (modifiedChunks.isEmpty()) { + return; + } + for (Chunk chunk : modifiedChunks) { + int chunkId = chunk.id; + meta.put(Chunk.getMetaKey(chunkId), chunk.asString()); + } + markMetaChanged(); + modifiedChunks.clear(); + } + } + + /** + * Shrink the file if possible, and if at least a given percentage can be + * saved. + * + * @param minPercent the minimum percentage to save + */ + private void shrinkFileIfPossible(int minPercent) { + if (fileStore.isReadOnly()) { + return; + } + long end = getFileLengthInUse(); + long fileSize = fileStore.size(); + if (end >= fileSize) { + return; + } + if (minPercent > 0 && fileSize - end < BLOCK_SIZE) { + return; + } + int savedPercent = (int) (100 - (end * 100 / fileSize)); + if (savedPercent < minPercent) { + return; + } + if (isOpenOrStopping()) { + sync(); + } + fileStore.truncate(end); + } + + /** + * Get the position right after the last used byte. + * + * @return the position + */ + private long getFileLengthInUse() { + long result = fileStore.getFileLengthInUse(); + assert result == measureFileLengthInUse() : result + " != " + measureFileLengthInUse(); + return result; + } + + /** + * Get the index of the first block after last occupied one. + * It marks the beginning of the last (infinite) free space. + * + * @return block index + */ + private long getAfterLastBlock() { + return fileStore.getAfterLastBlock(); + } + + private long measureFileLengthInUse() { + long size = 2; + for (Chunk c : chunks.values()) { + if (c.isSaved()) { + size = Math.max(size, c.block + c.len); + } + } + return size * BLOCK_SIZE; + } + + /** + * Check whether there are any unsaved changes. + * + * @return if there are any changes + */ + public boolean hasUnsavedChanges() { + if (metaChanged) { + return true; + } + for (MVMap m : maps.values()) { + if (!m.isClosed()) { + if(m.hasChangesSince(lastStoredVersion)) { + return true; + } + } + } + return false; + } + + private Chunk readChunkHeader(long block) { + long p = block * BLOCK_SIZE; + ByteBuffer buff = fileStore.readFully(p, Chunk.MAX_HEADER_LENGTH); + return Chunk.readChunkHeader(buff, p); + } + + private Chunk readChunkHeaderOptionally(long block) { + try { + Chunk chunk = readChunkHeader(block); + return chunk.block != block ? null : chunk; + } catch (Exception ignore) { + return null; + } + } + + private Chunk readChunkHeaderOptionally(long block, int expectedId) { + Chunk chunk = readChunkHeaderOptionally(block); + return chunk == null || chunk.id != expectedId ? null : chunk; + } + + /** + * Compact by moving all chunks next to each other. + */ + public void compactMoveChunks() { + compactMoveChunks(100, Long.MAX_VALUE); + } + + /** + * Compact the store by moving all chunks next to each other, if there is + * free space between chunks. This might temporarily increase the file size. + * Chunks are overwritten irrespective of the current retention time. Before + * overwriting chunks and before resizing the file, syncFile() is called. + * + * @param targetFillRate do nothing if the file store fill rate is higher + * than this + * @param moveSize the number of bytes to move + */ + private void compactMoveChunks(int targetFillRate, long moveSize) { + storeLock.lock(); + try { + checkOpen(); + if (lastChunk != null && reuseSpace) { + int oldRetentionTime = retentionTime; + boolean oldReuse = reuseSpace; + try { + retentionTime = -1; + if (getFillRate() <= targetFillRate) { + compactMoveChunks(moveSize); + } + } finally { + reuseSpace = oldReuse; + retentionTime = oldRetentionTime; + } + } + } finally { + storeLock.unlock(); + } + } + + private boolean compactMoveChunks(long moveSize) { + dropUnusedChunks(); + long start = fileStore.getFirstFree() / BLOCK_SIZE; + Iterable chunksToMove = findChunksToMove(start, moveSize); + if (chunksToMove == null) { + return false; + } + compactMoveChunks(chunksToMove); + return true; + } + + private Iterable findChunksToMove(long startBlock, long moveSize) { + long maxBlocksToMove = moveSize / BLOCK_SIZE; + Iterable result = null; + if (maxBlocksToMove > 0) { + PriorityQueue queue = new PriorityQueue<>(chunks.size() / 2 + 1, + new Comparator() { + @Override + public int compare(Chunk o1, Chunk o2) { + // instead of selection just closest to beginning of the file, + // pick smaller chunk(s) which sit in between bigger holes + int res = Integer.compare(o2.collectPriority, o1.collectPriority); + if (res != 0) { + return res; + } + return Long.signum(o2.block - o1.block); + } + }); + long size = 0; + for (Chunk chunk : chunks.values()) { + if (chunk.isSaved() && chunk.block > startBlock) { + chunk.collectPriority = getMovePriority(chunk); + queue.offer(chunk); + size += chunk.len; + while (size > maxBlocksToMove) { + Chunk removed = queue.poll(); + if (removed == null) { + break; + } + size -= removed.len; + } + } + } + if (!queue.isEmpty()) { + ArrayList list = new ArrayList<>(queue); + Collections.sort(list, Chunk.PositionComparator.INSTANCE); + result = list; + } + } + return result; + } + + private int getMovePriority(Chunk chunk) { + return fileStore.getMovePriority((int)chunk.block); + } + + private void compactMoveChunks(Iterable move) { + assert storeLock.isHeldByCurrentThread(); + if (move != null) { + assert lastChunk != null; + // this will ensure better recognition of the last chunk + // in case of power failure, since we are going to move older chunks + // to the end of the file + writeStoreHeader(); + sync(); + + Iterator iterator = move.iterator(); + assert iterator.hasNext(); + long leftmostBlock = iterator.next().block; + long originalBlockCount = getAfterLastBlock(); + // we need to ensure that chunks moved within the following loop + // do not overlap with space just released by chunks moved before them, + // hence the need to reserve this area [leftmostBlock, originalBlockCount) + for (Chunk chunk : move) { + moveChunk(chunk, leftmostBlock, originalBlockCount); + } + // update the metadata (hopefully within the file) + store(leftmostBlock, originalBlockCount); + sync(); + + Chunk chunkToMove = lastChunk; + long postEvacuationBlockCount = getAfterLastBlock(); + + boolean chunkToMoveIsAlreadyInside = chunkToMove.block < leftmostBlock; + boolean movedToEOF = !chunkToMoveIsAlreadyInside; + // move all chunks, which previously did not fit before reserved area + // now we can re-use previously reserved area [leftmostBlock, originalBlockCount), + // but need to reserve [originalBlockCount, postEvacuationBlockCount) + for (Chunk c : move) { + if (c.block >= originalBlockCount && + moveChunk(c, originalBlockCount, postEvacuationBlockCount)) { + assert c.block < originalBlockCount; + movedToEOF = true; + } + } + assert postEvacuationBlockCount >= getAfterLastBlock(); + + if (movedToEOF) { + boolean moved = moveChunkInside(chunkToMove, originalBlockCount); + + // store a new chunk with updated metadata (hopefully within a file) + store(originalBlockCount, postEvacuationBlockCount); + sync(); + // if chunkToMove did not fit within originalBlockCount (move is + // false), and since now previously reserved area + // [originalBlockCount, postEvacuationBlockCount) also can be + // used, lets try to move that chunk into this area, closer to + // the beginning of the file + long lastBoundary = moved || chunkToMoveIsAlreadyInside ? + postEvacuationBlockCount : chunkToMove.block; + moved = !moved && moveChunkInside(chunkToMove, lastBoundary); + if (moveChunkInside(lastChunk, lastBoundary) || moved) { + store(lastBoundary, -1); + } + } + + shrinkFileIfPossible(0); + sync(); + } + } + + private boolean moveChunkInside(Chunk chunkToMove, long boundary) { + boolean res = chunkToMove.block >= boundary && + fileStore.predictAllocation(chunkToMove.len, boundary, -1) < boundary && + moveChunk(chunkToMove, boundary, -1); + assert !res || chunkToMove.block + chunkToMove.len <= boundary; + return res; + } + + /** + * Move specified chunk into free area of the file. "Reserved" area + * specifies file interval to be avoided, when un-allocated space will be + * chosen for a new chunk's location. + * + * @param chunk to move + * @param reservedAreaLow low boundary of reserved area, inclusive + * @param reservedAreaHigh high boundary of reserved area, exclusive + * @return true if block was moved, false otherwise + */ + private boolean moveChunk(Chunk chunk, long reservedAreaLow, long reservedAreaHigh) { + // ignore if already removed during the previous store operations + // those are possible either as explicit commit calls + // or from meta map updates at the end of this method + if (!chunks.containsKey(chunk.id)) { + return false; + } + WriteBuffer buff = getWriteBuffer(); + long start = chunk.block * BLOCK_SIZE; + int length = chunk.len * BLOCK_SIZE; + buff.limit(length); + ByteBuffer readBuff = fileStore.readFully(start, length); + Chunk chunkFromFile = Chunk.readChunkHeader(readBuff, start); + int chunkHeaderLen = readBuff.position(); + buff.position(chunkHeaderLen); + buff.put(readBuff); + long pos = fileStore.allocate(length, reservedAreaLow, reservedAreaHigh); + long block = pos / BLOCK_SIZE; + // in the absence of a reserved area, + // block should always move closer to the beginning of the file + assert reservedAreaHigh > 0 || block <= chunk.block : block + " " + chunk; + buff.position(0); + // can not set chunk's new block/len until it's fully written at new location, + // because concurrent reader can pick it up prematurely, + // also occupancy accounting fields should not leak into header + chunkFromFile.block = block; + chunkFromFile.next = 0; + chunkFromFile.writeChunkHeader(buff, chunkHeaderLen); + buff.position(length - Chunk.FOOTER_LENGTH); + buff.put(chunkFromFile.getFooterBytes()); + buff.position(0); + write(pos, buff.getBuffer()); + releaseWriteBuffer(buff); + fileStore.free(start, length); + chunk.block = block; + chunk.next = 0; + meta.put(Chunk.getMetaKey(chunk.id), chunk.asString()); + markMetaChanged(); + return true; + } + + /** + * Force all stored changes to be written to the storage. The default + * implementation calls FileChannel.force(true). + */ + public void sync() { + checkOpen(); + FileStore f = fileStore; + if (f != null) { + f.sync(); + } + } + + /** + * Compact store file, that is, compact blocks that have a low + * fill rate, and move chunks next to each other. This will typically + * shrink the file. Changes are flushed to the file, and old + * chunks are overwritten. + * + * @param maxCompactTime the maximum time in milliseconds to compact + */ + public void compactFile(long maxCompactTime) { + setRetentionTime(0); + long start = System.nanoTime(); + while (compact(95, 16 * 1024 * 1024)) { + sync(); + compactMoveChunks(95, 16 * 1024 * 1024); + long time = System.nanoTime() - start; + if (time > TimeUnit.MILLISECONDS.toNanos(maxCompactTime)) { + break; + } + } + } + + /** + * Try to increase the fill rate by re-writing partially full chunks. Chunks + * with a low number of live items are re-written. + *

+ * If the current fill rate is higher than the target fill rate, nothing is + * done. + *

+ * Please note this method will not necessarily reduce the file size, as + * empty chunks are not overwritten. + *

+ * Only data of open maps can be moved. For maps that are not open, the old + * chunk is still referenced. Therefore, it is recommended to open all maps + * before calling this method. + * + * @param targetFillRate the minimum percentage of live entries + * @param write the minimum number of bytes to write + * @return if a chunk was re-written + */ + public boolean compact(int targetFillRate, int write) { + if (reuseSpace && lastChunk != null) { + checkOpen(); + if (targetFillRate > 0 && getChunksFillRate() < targetFillRate) { + // We can't wait forever for the lock here, + // because if called from the background thread, + // it might go into deadlock with concurrent database closure + // and attempt to stop this thread. + try { + if (storeLock.tryLock(10, TimeUnit.MILLISECONDS)) { + try { + return rewriteChunks(write); + } finally { + storeLock.unlock(); + } + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + return false; + } + + private boolean rewriteChunks(int writeLimit) { + TxCounter txCounter = registerVersionUsage(); + try { + Iterable old = findOldChunks(writeLimit); + if (old != null) { + HashSet idSet = createIdSet(old); + return !idSet.isEmpty() && compactRewrite(idSet) > 0; + } + } finally { + deregisterVersionUsage(txCounter); + } + return false; + } + + /** + * Get the current fill rate (percentage of used space in the file). Unlike + * the fill rate of the store, here we only account for chunk data; the fill + * rate here is how much of the chunk data is live (still referenced). Young + * chunks are considered live. + * + * @return the fill rate, in percent (100 is completely full) + */ + public int getChunksFillRate() { + long maxLengthSum = 1; + long maxLengthLiveSum = 1; + for (Chunk c : chunks.values()) { + assert c.maxLen >= 0; + maxLengthSum += c.maxLen; + maxLengthLiveSum += c.maxLenLive; + } + // the fill rate of all chunks combined + int fillRate = (int) (100 * maxLengthLiveSum / maxLengthSum); + return fillRate; + } + + private int getProjectedFillRate() { + int vacatedBlocks = 0; + long maxLengthSum = 1; + long maxLengthLiveSum = 1; + long time = getTimeSinceCreation(); + for (Chunk c : chunks.values()) { + assert c.maxLen >= 0; + if (isRewritable(c, time)) { + assert c.maxLenLive >= c.maxLenLive; + vacatedBlocks += c.len; + maxLengthSum += c.maxLen; + maxLengthLiveSum += c.maxLenLive; + } + } + int additionalBlocks = (int) (vacatedBlocks * maxLengthLiveSum / maxLengthSum); + int fillRate = fileStore.getProjectedFillRate(vacatedBlocks - additionalBlocks); + return fillRate; + } + + public int getFillRate() { + return fileStore.getFillRate(); + } + + private Iterable findOldChunks(int writeLimit) { + assert lastChunk != null; + long time = getTimeSinceCreation(); + + // the queue will contain chunks we want to free up + PriorityQueue queue = new PriorityQueue<>(this.chunks.size() / 4 + 1, + new Comparator() { + @Override + public int compare(Chunk o1, Chunk o2) { + int comp = Integer.compare(o2.collectPriority, o1.collectPriority); + if (comp == 0) { + comp = Long.compare(o2.maxLenLive, o2.maxLenLive); + } + return comp; + } + }); + + long totalSize = 0; + long latestVersion = lastChunk.version + 1; + for (Chunk chunk : chunks.values()) { + // only look at chunk older than the retention time + // (it's possible to compact chunks earlier, but right + // now we don't do that) + if (isRewritable(chunk, time)) { + long age = latestVersion - chunk.version; + chunk.collectPriority = (int) (chunk.getFillRate() * 1000 / age); + totalSize += chunk.maxLenLive; + queue.offer(chunk); + while (totalSize > writeLimit) { + Chunk removed = queue.poll(); + if (removed == null) { + break; + } + totalSize -= removed.maxLenLive; + } + } + } + + return queue.isEmpty() ? null : queue; + } + + private boolean isRewritable(Chunk chunk, long time) { + return chunk.isRewritable() && isSeasonedChunk(chunk, time); + } + + private int compactRewrite(Set set) { + assert storeLock.isHeldByCurrentThread(); + // this will ensure better recognition of the last chunk + // in case of power failure, since we are going to move older chunks + // to the end of the file + writeStoreHeader(); + sync(); + + int rewrittenPageCount = 0; + storeLock.unlock(); + try { + for (MVMap map : maps.values()) { + if (!map.isClosed() && !map.isSingleWriter()) { + try { + rewrittenPageCount += map.rewrite(set); + } catch(IllegalStateException ex) { + if (!map.isClosed()) { + throw ex; + } + } + } + } + int rewriteMetaCount = meta.rewrite(set); + if (rewriteMetaCount > 0) { + markMetaChanged(); + rewrittenPageCount += rewriteMetaCount; + } + } finally { + storeLock.lock(); + } + commit(); + assert validateRewrite(set); + return rewrittenPageCount; + } + + private boolean validateRewrite(Set set) { + for (Integer chunkId : set) { + Chunk chunk = chunks.get(chunkId); + if (chunk != null && chunk.isLive()) { + int pageCountLive = chunk.pageCountLive; + RemovedPageInfo[] removedPageInfos = removedPages.toArray(new RemovedPageInfo[0]); + for (RemovedPageInfo rpi : removedPageInfos) { + if (rpi.getPageChunkId() == chunk.id) { + --pageCountLive; + } + } + if (pageCountLive != 0) { + for (String mapName : getMapNames()) { + if (!mapName.startsWith("undoLog") && hasData(mapName)) { // non-singleWriter map has data + int mapId = getMapId(mapName); + if (!maps.containsKey(mapId)) { // map is not open + // all bets are off + return true; + } + } + } + assert pageCountLive != 0 : chunk + " " + Arrays.toString(removedPageInfos); + } + } + } + return true; + } + + private static HashSet createIdSet(Iterable toCompact) { + HashSet set = new HashSet<>(); + for (Chunk c : toCompact) { + set.add(c.id); + } + return set; + } + + /** + * Read a page. + * + * @param map the map + * @param pos the page position + * @return the page + */ + Page readPage(MVMap map, long pos) { + try { + if (!DataUtils.isPageSaved(pos)) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_CORRUPT, "Position 0"); + } + Page p = cache == null ? null : cache.get(pos); + if (p == null) { + ByteBuffer buff = readBufferForPage(pos, map.getId()); + try { + p = Page.read(buff, pos, map); + } catch (Exception e) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT, + "Unable to read the page at position {0}", pos, e); + } + cachePage(p); + } + return p; + } catch (IllegalStateException e) { + if (recoveryMode) { + return map.createEmptyLeaf(); + } + throw e; + } + } + + /** + * Remove a page. + * + * @param pos the position of the page + * @param version at which page was removed + * @param pinned whether page is considered pinned + */ + void accountForRemovedPage(long pos, long version, boolean pinned) { + assert DataUtils.isPageSaved(pos); + RemovedPageInfo rpi = new RemovedPageInfo(pos, pinned, version); + removedPages.add(rpi); + } + + Compressor getCompressorFast() { + if (compressorFast == null) { + compressorFast = new CompressLZF(); + } + return compressorFast; + } + + Compressor getCompressorHigh() { + if (compressorHigh == null) { + compressorHigh = new CompressDeflate(); + } + return compressorHigh; + } + + int getCompressionLevel() { + return compressionLevel; + } + + public int getPageSplitSize() { + return pageSplitSize; + } + + public int getKeysPerPage() { + return keysPerPage; + } + + public long getMaxPageSize() { + return cache == null ? Long.MAX_VALUE : cache.getMaxItemSize() >> 4; + } + + public boolean getReuseSpace() { + return reuseSpace; + } + + /** + * Whether empty space in the file should be re-used. If enabled, old data + * is overwritten (default). If disabled, writes are appended at the end of + * the file. + *

+ * This setting is specially useful for online backup. To create an online + * backup, disable this setting, then copy the file (starting at the + * beginning of the file). In this case, concurrent backup and write + * operations are possible (obviously the backup process needs to be faster + * than the write operations). + * + * @param reuseSpace the new value + */ + public void setReuseSpace(boolean reuseSpace) { + this.reuseSpace = reuseSpace; + } + + public int getRetentionTime() { + return retentionTime; + } + + /** + * How long to retain old, persisted chunks, in milliseconds. Chunks that + * are older may be overwritten once they contain no live data. + *

+ * The default value is 45000 (45 seconds) when using the default file + * store. It is assumed that a file system and hard disk will flush all + * write buffers within this time. Using a lower value might be dangerous, + * unless the file system and hard disk flush the buffers earlier. To + * manually flush the buffers, use + * MVStore.getFile().force(true), however please note that + * according to various tests this does not always work as expected + * depending on the operating system and hardware. + *

+ * The retention time needs to be long enough to allow reading old chunks + * while traversing over the entries of a map. + *

+ * This setting is not persisted. + * + * @param ms how many milliseconds to retain old chunks (0 to overwrite them + * as early as possible) + */ + public void setRetentionTime(int ms) { + this.retentionTime = ms; + } + + /** + * How many versions to retain for in-memory stores. If not set, 5 old + * versions are retained. + * + * @param count the number of versions to keep + */ + public void setVersionsToKeep(int count) { + this.versionsToKeep = count; + } + + /** + * Get the oldest version to retain in memory (for in-memory stores). + * + * @return the version + */ + public long getVersionsToKeep() { + return versionsToKeep; + } + + /** + * Get the oldest version to retain. + * We keep at least number of previous versions specified by "versionsToKeep" + * configuration parameter (default 5). + * Previously it was used only in case of non-persistent MVStore. + * Now it's honored in all cases (although H2 always sets it to zero). + * Oldest version determination also takes into account calls (de)registerVersionUsage(), + * an will not release the version, while version is still in use. + * + * @return the version + */ + long getOldestVersionToKeep() { + long v = oldestVersionToKeep.get(); + v = Math.max(v - versionsToKeep, INITIAL_VERSION); + if (fileStore != null) { + long storeVersion = lastStoredVersion; + if (storeVersion != INITIAL_VERSION && storeVersion < v) { + v = storeVersion; + } + } + return v; + } + + private void setOldestVersionToKeep(long oldestVersionToKeep) { + boolean success; + do { + long current = this.oldestVersionToKeep.get(); + // Oldest version may only advance, never goes back + success = oldestVersionToKeep <= current || + this.oldestVersionToKeep.compareAndSet(current, oldestVersionToKeep); + } while (!success); + } + + /** + * Check whether all data can be read from this version. This requires that + * all chunks referenced by this version are still available (not + * overwritten). + * + * @param version the version + * @return true if all data can be read + */ + private boolean isKnownVersion(long version) { + if (version > currentVersion || version < 0) { + return false; + } + if (version == currentVersion || chunks.isEmpty()) { + // no stored data + return true; + } + // need to check if a chunk for this version exists + Chunk c = getChunkForVersion(version); + if (c == null) { + return false; + } + // also, all chunks referenced by this version + // need to be available in the file + MVMap oldMeta = getMetaMap(version); + try { + for (Iterator it = oldMeta.keyIterator(DataUtils.META_CHUNK); it.hasNext();) { + String chunkKey = it.next(); + if (!chunkKey.startsWith(DataUtils.META_CHUNK)) { + break; + } + if (!meta.containsKey(chunkKey)) { + String s = oldMeta.get(chunkKey); + Chunk c2 = Chunk.fromString(s); + Chunk test = readChunkHeaderAndFooter(c2.block, c2.id); + if (test == null) { + return false; + } + } + } + } catch (IllegalStateException e) { + // the chunk missing where the metadata is stored + return false; + } + return true; + } + + /** + * Adjust amount of "unsaved memory" meaning amount of RAM occupied by pages + * not saved yet to the file. This is the amount which triggers auto-commit. + * + * @param memory adjustment + */ + public void registerUnsavedMemory(int memory) { + // this counter was intentionally left unprotected against race + // condition for performance reasons + // TODO: evaluate performance impact of atomic implementation, + // since updates to unsavedMemory are largely aggregated now + unsavedMemory += memory; + int newValue = unsavedMemory; + if (newValue > autoCommitMemory && autoCommitMemory > 0) { + saveNeeded = true; + } + } + + boolean isSaveNeeded() { + return saveNeeded; + } + + /** + * This method is called before writing to a map. + * + * @param map the map + */ + void beforeWrite(MVMap map) { + if (saveNeeded && fileStore != null && isOpenOrStopping() && + // condition below is to prevent potential deadlock, + // because we should never seek storeLock while holding + // map root lock + (storeLock.isHeldByCurrentThread() || !map.getRoot().isLockedByCurrentThread()) && + // to avoid infinite recursion via store() -> dropUnusedChunks() -> meta.remove() + map != meta) { + + saveNeeded = false; + // check again, because it could have been written by now + if (unsavedMemory > autoCommitMemory && autoCommitMemory > 0) { + // if unsaved memory creation rate is to high, + // some back pressure need to be applied + // to slow things down and avoid OOME + if (3 * unsavedMemory > 4 * autoCommitMemory && !map.isSingleWriter()) { + commit(); + } else { + tryCommit(); + } + } + } + } + + /** + * Get the store version. The store version is usually used to upgrade the + * structure of the store after upgrading the application. Initially the + * store version is 0, until it is changed. + * + * @return the store version + */ + public int getStoreVersion() { + checkOpen(); + String x = meta.get("setting.storeVersion"); + return x == null ? 0 : DataUtils.parseHexInt(x); + } + + /** + * Update the store version. + * + * @param version the new store version + */ + public void setStoreVersion(int version) { + storeLock.lock(); + try { + checkOpen(); + markMetaChanged(); + meta.put("setting.storeVersion", Integer.toHexString(version)); + } finally { + storeLock.unlock(); + } + } + + /** + * Revert to the beginning of the current version, reverting all uncommitted + * changes. + */ + public void rollback() { + rollbackTo(currentVersion); + } + + /** + * Revert to the beginning of the given version. All later changes (stored + * or not) are forgotten. All maps that were created later are closed. A + * rollback to a version before the last stored version is immediately + * persisted. Rollback to version 0 means all data is removed. + * + * @param version the version to revert to + */ + public void rollbackTo(long version) { + storeLock.lock(); + try { + checkOpen(); + if (version == 0) { + // special case: remove all data + meta.setInitialRoot(meta.createEmptyLeaf(), INITIAL_VERSION); + deadChunks.clear(); + removedPages.clear(); + chunks.clear(); + clearCaches(); + if (fileStore != null) { + fileStore.clear(); + } + lastChunk = null; + versions.clear(); + currentVersion = version; + setWriteVersion(version); + metaChanged = false; + lastStoredVersion = INITIAL_VERSION; + for (MVMap m : maps.values()) { + m.close(); + } + return; + } + DataUtils.checkArgument( + isKnownVersion(version), + "Unknown version {0}", version); + + TxCounter txCounter; + while ((txCounter = versions.peekLast()) != null && txCounter.version >= version) { + versions.removeLast(); + } + currentTxCounter = new TxCounter(version); + + meta.rollbackTo(version); + metaChanged = false; + // find out which chunks to remove, + // and which is the newest chunk to keep + // (the chunk list can have gaps) + ArrayList remove = new ArrayList<>(); + Chunk keep = null; + for (Chunk c : chunks.values()) { + if (c.version > version) { + remove.add(c.id); + } else if (keep == null || keep.version < c.version) { + keep = c; + } + } + if (!remove.isEmpty()) { + // remove the youngest first, so we don't create gaps + // (in case we remove many chunks) + Collections.sort(remove, Collections.reverseOrder()); + for (int id : remove) { + Chunk c = chunks.remove(id); + if (c != null) { + long start = c.block * BLOCK_SIZE; + int length = c.len * BLOCK_SIZE; + freeFileSpace(start, length); + // overwrite the chunk, + // so it is not be used later on + WriteBuffer buff = getWriteBuffer(); + buff.limit(length); + // buff.clear() does not set the data + Arrays.fill(buff.getBuffer().array(), (byte) 0); + write(start, buff.getBuffer()); + releaseWriteBuffer(buff); + // only really needed if we remove many chunks, when writes are + // re-ordered - but we do it always, because rollback is not + // performance critical + sync(); + } + } + lastChunk = keep; + writeStoreHeader(); + readStoreHeader(); + } + deadChunks.clear(); + removedPages.clear(); + clearCaches(); + currentVersion = version; + if (lastStoredVersion == INITIAL_VERSION) { + lastStoredVersion = currentVersion - 1; + } + for (MVMap m : new ArrayList<>(maps.values())) { + int id = m.getId(); + if (m.getCreateVersion() >= version) { + m.close(); + maps.remove(id); + } else { + if (!m.rollbackRoot(version)) { + m.setRootPos(getRootPos(meta, id), version); + } + } + } + } finally { + storeLock.unlock(); + } + } + + private void clearCaches() { + if (cache != null) { + cache.clear(); + } + } + + private static long getRootPos(MVMap map, int mapId) { + String root = map.get(MVMap.getMapRootKey(mapId)); + return root == null ? 0 : DataUtils.parseHexLong(root); + } + + /** + * Get the current version of the data. When a new store is created, the + * version is 0. + * + * @return the version + */ + public long getCurrentVersion() { + return currentVersion; + } + + /** + * Get the file store. + * + * @return the file store + */ + public FileStore getFileStore() { + return fileStore; + } + + /** + * Get the store header. This data is for informational purposes only. The + * data is subject to change in future versions. The data should not be + * modified (doing so may corrupt the store). + * + * @return the store header + */ + public Map getStoreHeader() { + return storeHeader; + } + + private void checkOpen() { + if (!isOpenOrStopping()) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_CLOSED, + "This store is closed", panicException); + } + } + + /** + * Rename a map. + * + * @param map the map + * @param newName the new name + */ + public void renameMap(MVMap map, String newName) { + checkOpen(); + DataUtils.checkArgument(map != meta, + "Renaming the meta map is not allowed"); + int id = map.getId(); + String oldName = getMapName(id); + if (oldName != null && !oldName.equals(newName)) { + String idHexStr = Integer.toHexString(id); + // at first create a new name as an "alias" + String existingIdHexStr = meta.putIfAbsent(DataUtils.META_NAME + newName, idHexStr); + // we need to cope with the case of previously unfinished rename + DataUtils.checkArgument( + existingIdHexStr == null || existingIdHexStr.equals(idHexStr), + "A map named {0} already exists", newName); + // switch roles of a new and old names - old one is an alias now + meta.put(MVMap.getMapKey(id), map.asString(newName)); + // get rid of the old name completely + meta.remove(DataUtils.META_NAME + oldName); + markMetaChanged(); + } + } + + /** + * Remove a map from the current version of the store. + * + * @param map the map to remove + */ + public void removeMap(MVMap map) { + storeLock.lock(); + try { + checkOpen(); + DataUtils.checkArgument(map != meta, + "Removing the meta map is not allowed"); + RootReference rootReference = map.clearIt(); + map.close(); + + updateCounter += rootReference.updateCounter; + updateAttemptCounter += rootReference.updateAttemptCounter; + + int id = map.getId(); + String name = getMapName(id); + if (meta.remove(MVMap.getMapKey(id)) != null) { + markMetaChanged(); + } + if (meta.remove(DataUtils.META_NAME + name) != null) { + markMetaChanged(); + } + } finally { + storeLock.unlock(); + } + } + + /** + * Performs final stage of map removal - delete root location info from the meta table. + * Map is supposedly closed and anonymous and has no outstanding usage by now. + * + * @param mapId to deregister + */ + void deregisterMapRoot(int mapId) { + if (meta.remove(MVMap.getMapRootKey(mapId)) != null) { + markMetaChanged(); + } + } + + /** + * Remove map by name. + * + * @param name the map name + */ + public void removeMap(String name) { + int id = getMapId(name); + if(id > 0) { + MVMap map = getMap(id); + if (map == null) { + map = openMap(name); + } + removeMap(map); + } + } + + /** + * Get the name of the given map. + * + * @param id the map id + * @return the name, or null if not found + */ + public String getMapName(int id) { + checkOpen(); + String m = meta.get(MVMap.getMapKey(id)); + return m == null ? null : DataUtils.getMapName(m); + } + + private int getMapId(String name) { + String m = meta.get(DataUtils.META_NAME + name); + return m == null ? -1 : DataUtils.parseHexInt(m); + } + + /** + * Commit and save all changes, if there are any, and compact the store if + * needed. + */ + void writeInBackground() { + try { + if (!isOpenOrStopping() || isReadOnly()) { + return; + } + + // could also commit when there are many unsaved pages, + // but according to a test it doesn't really help + + long time = getTimeSinceCreation(); + if (time > lastCommitTime + autoCommitDelay) { + tryCommit(); + if (autoCompactFillRate < 0) { + compact(-getTargetFillRate(), autoCommitMemory); + } + } + int targetFillRate; + int projectedFillRate; + if (isIdle()) { + doMaintenance(autoCompactFillRate); + } else if (fileStore.isFragmented()) { + if (storeLock.tryLock(10, TimeUnit.MILLISECONDS)) { + try { + compactMoveChunks(autoCommitMemory * 4); + } finally { + storeLock.unlock(); + } + } + } else if (lastChunk != null && getFillRate() > (targetFillRate = getTargetFillRate()) + && (projectedFillRate = getProjectedFillRate()) < targetFillRate) { + if (storeLock.tryLock(10, TimeUnit.MILLISECONDS)) { + try { + int writeLimit = autoCommitMemory * targetFillRate / Math.max(projectedFillRate, 1); + if (rewriteChunks(writeLimit)) { + dropUnusedChunks(); + } + } finally { + storeLock.unlock(); + } + } + } + autoCompactLastFileOpCount = fileStore.getWriteCount() + fileStore.getReadCount(); + } catch (InterruptedException ignore) { + } catch (Throwable e) { + handleException(e); + if (backgroundExceptionHandler == null) { + throw e; + } + } + } + + private void doMaintenance(int targetFillRate) { + if (autoCompactFillRate > 0 && lastChunk != null && reuseSpace) { + try { + int lastProjectedFillRate = -1; + for (int cnt = 0; cnt < 5; cnt++) { + int fillRate = getFillRate(); + int projectedFillRate = fillRate; + if (fillRate > targetFillRate) { + projectedFillRate = getProjectedFillRate(); + if (projectedFillRate > targetFillRate || projectedFillRate <= lastProjectedFillRate) { + break; + } + } + lastProjectedFillRate = projectedFillRate; + // We can't wait forever for the lock here, + // because if called from the background thread, + // it might go into deadlock with concurrent database closure + // and attempt to stop this thread. + if (!storeLock.tryLock(10, TimeUnit.MILLISECONDS)) { + break; + } + try { + int writeLimit = autoCommitMemory * targetFillRate / Math.max(projectedFillRate, 1); + if (projectedFillRate < fillRate) { + if ((!rewriteChunks(writeLimit) || dropUnusedChunks() == 0) && cnt > 0) { + break; + } + } + if (!compactMoveChunks(writeLimit)) { + break; + } + } finally { + storeLock.unlock(); + } + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + private int getTargetFillRate() { + int targetRate = autoCompactFillRate; + // use a lower fill rate if there were any file operations since the last time + if (!isIdle()) { + targetRate /= 3; + } + return targetRate; + } + + private boolean isIdle() { + return autoCompactLastFileOpCount == fileStore.getWriteCount() + fileStore.getReadCount(); + } + + private void handleException(Throwable ex) { + if (backgroundExceptionHandler != null) { + try { + backgroundExceptionHandler.uncaughtException(Thread.currentThread(), ex); + } catch(Throwable ignore) { + if (ex != ignore) { // OOME may be the same + ex.addSuppressed(ignore); + } + } + } + } + + /** + * Set the read cache size in MB. + * + * @param mb the cache size in MB. + */ + public void setCacheSize(int mb) { + final long bytes = (long) mb * 1024 * 1024; + if (cache != null) { + cache.setMaxMemory(bytes); + cache.clear(); + } + } + + private boolean isOpen() { + return state == STATE_OPEN; + } + + /** + * Determine that store is open, or wait for it to be closed (by other thread) + * @return true if store is open, false otherwise + */ + public boolean isClosed() { + if (isOpen()) { + return false; + } + storeLock.lock(); + try { + assert state == STATE_CLOSED; + return true; + } finally { + storeLock.unlock(); + } + } + + private boolean isOpenOrStopping() { + return state <= STATE_STOPPING; + } + + private void stopBackgroundThread(boolean waitForIt) { + // Loop here is not strictly necessary, except for case of a spurious failure, + // which should not happen with non-weak flavour of CAS operation, + // but I've seen it, so just to be safe... + BackgroundWriterThread t; + while ((t = backgroundWriterThread.get()) != null) { + if (backgroundWriterThread.compareAndSet(t, null)) { + // if called from within the thread itself - can not join + if (t != Thread.currentThread()) { + synchronized (t.sync) { + t.sync.notifyAll(); + } + + if (waitForIt) { + try { + t.join(); + } catch (Exception e) { + // ignore + } + } + } + break; + } + } + } + + /** + * Set the maximum delay in milliseconds to auto-commit changes. + *

+ * To disable auto-commit, set the value to 0. In this case, changes are + * only committed when explicitly calling commit. + *

+ * The default is 1000, meaning all changes are committed after at most one + * second. + * + * @param millis the maximum delay + */ + public void setAutoCommitDelay(int millis) { + if (autoCommitDelay == millis) { + return; + } + autoCommitDelay = millis; + if (fileStore == null || fileStore.isReadOnly()) { + return; + } + stopBackgroundThread(true); + // start the background thread if needed + if (millis > 0 && isOpen()) { + int sleep = Math.max(1, millis / 10); + BackgroundWriterThread t = + new BackgroundWriterThread(this, sleep, + fileStore.toString()); + if (backgroundWriterThread.compareAndSet(null, t)) { + t.start(); + } + } + } + + public boolean isBackgroundThread() { + return Thread.currentThread() == backgroundWriterThread.get(); + } + + /** + * Get the auto-commit delay. + * + * @return the delay in milliseconds, or 0 if auto-commit is disabled. + */ + public int getAutoCommitDelay() { + return autoCommitDelay; + } + + /** + * Get the maximum memory (in bytes) used for unsaved pages. If this number + * is exceeded, unsaved changes are stored to disk. + * + * @return the memory in bytes + */ + public int getAutoCommitMemory() { + return autoCommitMemory; + } + + /** + * Get the estimated memory (in bytes) of unsaved data. If the value exceeds + * the auto-commit memory, the changes are committed. + *

+ * The returned value is an estimation only. + * + * @return the memory in bytes + */ + public int getUnsavedMemory() { + return unsavedMemory; + } + + /** + * Put the page in the cache. + * @param page the page + */ + void cachePage(Page page) { + if (cache != null) { + cache.put(page.getPos(), page, page.getMemory()); + } + } + + /** + * Get the amount of memory used for caching, in MB. + * Note that this does not include the page chunk references cache, which is + * 25% of the size of the page cache. + * + * @return the amount of memory used for caching + */ + public int getCacheSizeUsed() { + if (cache == null) { + return 0; + } + return (int) (cache.getUsedMemory() >> 20); + } + + /** + * Get the maximum cache size, in MB. + * Note that this does not include the page chunk references cache, which is + * 25% of the size of the page cache. + * + * @return the cache size + */ + public int getCacheSize() { + if (cache == null) { + return 0; + } + return (int) (cache.getMaxMemory() >> 20); + } + + /** + * Get the cache. + * + * @return the cache + */ + public CacheLongKeyLIRS getCache() { + return cache; + } + + /** + * Whether the store is read-only. + * + * @return true if it is + */ + public boolean isReadOnly() { + return fileStore != null && fileStore.isReadOnly(); + } + + public int getCacheHitRatio() { + if (cache == null) { + return 0; + } + long hits = cache.getHits(); + return (int) (100 * hits / (hits + cache.getMisses() + 1)); + } + + public double getUpdateFailureRatio() { + long updateCounter = this.updateCounter; + long updateAttemptCounter = this.updateAttemptCounter; + RootReference rootReference = meta.getRoot(); + updateCounter += rootReference.updateCounter; + updateAttemptCounter += rootReference.updateAttemptCounter; + for (MVMap map : maps.values()) { + RootReference root = map.getRoot(); + updateCounter += root.updateCounter; + updateAttemptCounter += root.updateAttemptCounter; + } + return updateAttemptCounter == 0 ? 0 : 1 - ((double)updateCounter / updateAttemptCounter); + } + + /** + * Register opened operation (transaction). + * This would increment usage counter for the current version. + * This version (and all after it) should not be dropped until all + * transactions involved are closed and usage counter goes to zero. + * @return TxCounter to be decremented when operation finishes (transaction closed). + */ + public TxCounter registerVersionUsage() { + TxCounter txCounter; + while(true) { + txCounter = currentTxCounter; + if(txCounter.incrementAndGet() > 0) { + return txCounter; + } + // The only way for counter to be negative + // if it was retrieved right before onVersionChange() + // and now onVersionChange() is done. + // This version is eligible for reclamation now + // and should not be used here, so restore count + // not to upset accounting and try again with a new + // version (currentTxCounter should have changed). + assert txCounter != currentTxCounter : txCounter; + txCounter.decrementAndGet(); + } + } + + /** + * De-register (close) completed operation (transaction). + * This will decrement usage counter for the corresponding version. + * If counter reaches zero, that version (and all unused after it) + * can be dropped immediately. + * + * @param txCounter to be decremented, obtained from registerVersionUsage() + */ + public void deregisterVersionUsage(TxCounter txCounter) { + if(txCounter != null) { + if(txCounter.decrementAndGet() <= 0) { + if (storeLock.isHeldByCurrentThread()) { + dropUnusedVersions(); + } else if (storeLock.tryLock()) { + try { + dropUnusedVersions(); + } finally { + storeLock.unlock(); + } + } + } + } + } + + private void onVersionChange(long version) { + TxCounter txCounter = currentTxCounter; + assert txCounter.get() >= 0; + versions.add(txCounter); + currentTxCounter = new TxCounter(version); + txCounter.decrementAndGet(); + dropUnusedVersions(); + } + + private void dropUnusedVersions() { + assert storeLock.isHeldByCurrentThread(); + TxCounter txCounter; + while ((txCounter = versions.peek()) != null + && txCounter.get() < 0) { + versions.poll(); + } + setOldestVersionToKeep((txCounter != null ? txCounter : currentTxCounter).version); + } + + private int dropUnusedChunks() { + assert storeLock.isHeldByCurrentThread(); + int count = 0; + if (!deadChunks.isEmpty()) { + long oldestVersionToKeep = getOldestVersionToKeep(); + long time = getTimeSinceCreation(); + Chunk chunk; + while ((chunk = deadChunks.poll()) != null && + (isSeasonedChunk(chunk, time) && canOverwriteChunk(chunk, oldestVersionToKeep) || + // if chunk is not ready yet, put it back and exit + // since this deque is inbounded, offerFirst() always return true + !deadChunks.offerFirst(chunk))) { + + if (chunks.remove(chunk.id) != null) { + if (meta.remove(Chunk.getMetaKey(chunk.id)) != null) { + markMetaChanged(); + } + if (chunk.isSaved()) { + freeChunkSpace(chunk); + } + ++count; + } + } + } + return count; + } + + private void freeChunkSpace(Chunk chunk) { + long start = chunk.block * BLOCK_SIZE; + int length = chunk.len * BLOCK_SIZE; + freeFileSpace(start, length); + } + + private void freeFileSpace(long start, int length) { + fileStore.free(start, length); + assert validateFileLength(start + ":" + length); + } + + private boolean validateFileLength(String msg) { + assert fileStore.getFileLengthInUse() == measureFileLengthInUse() : + fileStore.getFileLengthInUse() + " != " + measureFileLengthInUse() + " " + msg; + return true; + } + + /** + * Class TxCounter is a simple data structure to hold version of the store + * along with the counter of open transactions, + * which are still operating on this version. + */ + public static final class TxCounter { + + /** + * Version of a store, this TxCounter is related to + */ + public final long version; + + /** + * Counter of outstanding operation on this version of a store + */ + private volatile int counter; + + private static final AtomicIntegerFieldUpdater counterUpdater = + AtomicIntegerFieldUpdater.newUpdater(TxCounter.class, "counter"); + + + TxCounter(long version) { + this.version = version; + } + + int get() { + return counter; + } + + /** + * Increment and get the counter value. + * + * @return the new value + */ + int incrementAndGet() { + return counterUpdater.incrementAndGet(this); + } + + /** + * Decrement and get the counter values. + * + * @return the new value + */ + int decrementAndGet() { + return counterUpdater.decrementAndGet(this); + } + + @Override + public String toString() { + return "v=" + version + " / cnt=" + counter; + } + } + + /** + * A background writer thread to automatically store changes from time to + * time. + */ + private static class BackgroundWriterThread extends Thread { + + public final Object sync = new Object(); + private final MVStore store; + private final int sleep; + + BackgroundWriterThread(MVStore store, int sleep, String fileStoreName) { + super("MVStore background writer " + fileStoreName); + this.store = store; + this.sleep = sleep; + setDaemon(true); + } + + @Override + public void run() { + while (store.isBackgroundThread()) { + synchronized (sync) { + try { + sync.wait(sleep); + } catch (InterruptedException ignore) { + } + } + if (!store.isBackgroundThread()) { + break; + } + store.writeInBackground(); + } + } + } + + private static class RemovedPageInfo implements Comparable + { + final long version; + final int removedPageInfo; + + RemovedPageInfo(long pagePos, boolean pinned, long version) { + this.removedPageInfo = createRemovedPageInfo(pagePos, pinned); + this.version = version; + } + + @Override + public int compareTo(RemovedPageInfo other) { + return Long.compare(version, other.version); + } + + int getPageChunkId() { + return removedPageInfo >>> 6; + } + + int getPageLength() { + return DataUtils.decodePageLength((removedPageInfo >> 1) & 0x1F); + } + + /** + * Find out if removed page was pinned (can not be evacuated to a new chunk). + * @return true if page has been pinned + */ + boolean isPinned() { + return (removedPageInfo & 1) == 1; + } + + /** + * Transforms saved page position into removed page info, by eliminating page offset + * and replacing "page type" bit with "pinned page" flag. + * 0 "pinned" flag + * 1-5 encoded page length + * 6-31 chunk id + * @param pagePos of the saved page + * @param isPinned whether page belong to a "single writer" map + * @return removed page info that contains chunk id, page length and pinned flag + */ + private static int createRemovedPageInfo(long pagePos, boolean isPinned) { + int result = ((int) (pagePos >>> 32)) & ~0x3F | ((int) pagePos) & 0x3E; + if (isPinned) { + result |= 1; + } + return result; + } + + @Override + public String toString() { + return "RemovedPageInfo{" + + "version=" + version + + ", chunk=" + getPageChunkId() + + ", len=" + getPageLength() + + (isPinned() ? ", pinned" : "") + + '}'; + } + } + + /** + * A builder for an MVStore. + */ + public static final class Builder { + + private final HashMap config; + + private Builder(HashMap config) { + this.config = config; + } + + /** + * Creates new instance of MVStore.Builder. + */ + public Builder() { + config = new HashMap<>(); + } + + private Builder set(String key, Object value) { + config.put(key, value); + return this; + } + + /** + * Disable auto-commit, by setting the auto-commit delay and auto-commit + * buffer size to 0. + * + * @return this + */ + public Builder autoCommitDisabled() { + // we have a separate config option so that + // no thread is started if the write delay is 0 + // (if we only had a setter in the MVStore, + // the thread would need to be started in any case) + //set("autoCommitBufferSize", 0); + return set("autoCommitDelay", 0); + } + + /** + * Set the size of the write buffer, in KB disk space (for file-based + * stores). Unless auto-commit is disabled, changes are automatically + * saved if there are more than this amount of changes. + *

+ * The default is 1024 KB. + *

+ * When the value is set to 0 or lower, data is not automatically + * stored. + * + * @param kb the write buffer size, in kilobytes + * @return this + */ + public Builder autoCommitBufferSize(int kb) { + return set("autoCommitBufferSize", kb); + } + + /** + * Set the auto-compact target fill rate. If the average fill rate (the + * percentage of the storage space that contains active data) of the + * chunks is lower, then the chunks with a low fill rate are re-written. + * Also, if the percentage of empty space between chunks is higher than + * this value, then chunks at the end of the file are moved. Compaction + * stops if the target fill rate is reached. + *

+ * The default value is 40 (40%). The value 0 disables auto-compacting. + *

+ * + * @param percent the target fill rate + * @return this + */ + public Builder autoCompactFillRate(int percent) { + return set("autoCompactFillRate", percent); + } + + /** + * Use the following file name. If the file does not exist, it is + * automatically created. The parent directory already must exist. + * + * @param fileName the file name + * @return this + */ + public Builder fileName(String fileName) { + return set("fileName", fileName); + } + + /** + * Encrypt / decrypt the file using the given password. This method has + * no effect for in-memory stores. The password is passed as a + * char array so that it can be cleared as soon as possible. Please note + * there is still a small risk that password stays in memory (due to + * Java garbage collection). Also, the hashed encryption key is kept in + * memory as long as the file is open. + * + * @param password the password + * @return this + */ + public Builder encryptionKey(char[] password) { + return set("encryptionKey", password); + } + + /** + * Open the file in read-only mode. In this case, a shared lock will be + * acquired to ensure the file is not concurrently opened in write mode. + *

+ * If this option is not used, the file is locked exclusively. + *

+ * Please note a store may only be opened once in every JVM (no matter + * whether it is opened in read-only or read-write mode), because each + * file may be locked only once in a process. + * + * @return this + */ + public Builder readOnly() { + return set("readOnly", 1); + } + + /** + * Open the file in recovery mode, where some errors may be ignored. + * + * @return this + */ + public Builder recoveryMode() { + return set("recoveryMode", 1); + } + + /** + * Set the read cache size in MB. The default is 16 MB. + * + * @param mb the cache size in megabytes + * @return this + */ + public Builder cacheSize(int mb) { + return set("cacheSize", mb); + } + + /** + * Set the read cache concurrency. The default is 16, meaning 16 + * segments are used. + * + * @param concurrency the cache concurrency + * @return this + */ + public Builder cacheConcurrency(int concurrency) { + return set("cacheConcurrency", concurrency); + } + + /** + * Compress data before writing using the LZF algorithm. This will save + * about 50% of the disk space, but will slow down read and write + * operations slightly. + *

+ * This setting only affects writes; it is not necessary to enable + * compression when reading, even if compression was enabled when + * writing. + * + * @return this + */ + public Builder compress() { + return set("compress", 1); + } + + /** + * Compress data before writing using the Deflate algorithm. This will + * save more disk space, but will slow down read and write operations + * quite a bit. + *

+ * This setting only affects writes; it is not necessary to enable + * compression when reading, even if compression was enabled when + * writing. + * + * @return this + */ + public Builder compressHigh() { + return set("compress", 2); + } + + /** + * Set the amount of memory a page should contain at most, in bytes, + * before it is split. The default is 16 KB for persistent stores and 4 + * KB for in-memory stores. This is not a limit in the page size, as + * pages with one entry can get larger. It is just the point where pages + * that contain more than one entry are split. + * + * @param pageSplitSize the page size + * @return this + */ + public Builder pageSplitSize(int pageSplitSize) { + return set("pageSplitSize", pageSplitSize); + } + + /** + * Set the listener to be used for exceptions that occur when writing in + * the background thread. + * + * @param exceptionHandler the handler + * @return this + */ + public Builder backgroundExceptionHandler( + Thread.UncaughtExceptionHandler exceptionHandler) { + return set("backgroundExceptionHandler", exceptionHandler); + } + + /** + * Use the provided file store instead of the default one. + *

+ * File stores passed in this way need to be open. They are not closed + * when closing the store. + *

+ * Please note that any kind of store (including an off-heap store) is + * considered a "persistence", while an "in-memory store" means objects + * are not persisted and fully kept in the JVM heap. + * + * @param store the file store + * @return this + */ + public Builder fileStore(FileStore store) { + return set("fileStore", store); + } + + /** + * Open the store. + * + * @return the opened store + */ + public MVStore open() { + return new MVStore(config); + } + + @Override + public String toString() { + return DataUtils.appendMap(new StringBuilder(), config).toString(); + } + + /** + * Read the configuration from a string. + * + * @param s the string representation + * @return the builder + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + public static Builder fromString(String s) { + // Cast from HashMap to HashMap is safe + return new Builder((HashMap) DataUtils.parseMap(s)); + } + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Page.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Page.java new file mode 100644 index 000000000..ceb9d57dc --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/Page.java @@ -0,0 +1,1538 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.compress.Compressor; +import org.dizitart.no2.mvstore.compat.v1.mvstore.type.DataType; +import org.h2.util.Utils; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; + +import static org.dizitart.no2.mvstore.compat.v1.mvstore.DataUtils.PAGE_TYPE_LEAF; +import static org.h2.engine.Constants.*; + +/** + * A page (a node or a leaf). + *

+ * For b-tree nodes, the key at a given index is larger than the largest key of + * the child at the same index. + *

+ * File format: + * page length (including length): int + * check value: short + * map id: varInt + * number of keys: varInt + * type: byte (0: leaf, 1: node; +2: compressed) + * compressed: bytes saved (varInt) + * keys + * leaf: values (one for each key) + * node: children (1 more than keys) + */ +public abstract class Page implements Cloneable { + /** + * Map this page belongs to + */ + public final MVMap map; + + /** + * Position of this page's saved image within a Chunk + * or 0 if this page has not been saved yet + * or 1 if this page has not been saved yet, but already removed + * This "removed" flag is to keep track of pages that concurrently + * changed while they are being stored, in which case the live bookkeeping + * needs to be aware of such cases. + * Field need to be volatile to avoid races between saving thread setting it + * and other thread reading it to access the page. + * On top of this update atomicity is required so removal mark and saved position + * can be set concurrently + */ + private volatile long pos; + + /** + * The last result of a find operation is cached. + */ + private int cachedCompare; + + /** + * The estimated memory used in persistent case, IN_MEMORY marker value otherwise. + */ + private int memory; + + /** + * Amount of used disk space by this page only in persistent case. + */ + private int diskSpaceUsed; + + /** + * The keys. + */ + private Object[] keys; + + /** + * Updater for pos field, which can be updated when page is saved, + * but can be concurrently marked as removed + */ + private static final AtomicLongFieldUpdater posUpdater = AtomicLongFieldUpdater.newUpdater(Page.class, "pos"); + /** + * The estimated number of bytes used per child entry. + */ + static final int PAGE_MEMORY_CHILD = MEMORY_POINTER + 16; // 16 = two longs + + /** + * The estimated number of bytes used per base page. + */ + private static final int PAGE_MEMORY = MEMORY_OBJECT + // this + 2 * MEMORY_POINTER + // map, keys + MEMORY_ARRAY + // Object[] keys + 17; // pos, cachedCompare, memory, removedInMemory + /** + * The estimated number of bytes used per empty internal page object. + */ + static final int PAGE_NODE_MEMORY = PAGE_MEMORY + // super + MEMORY_POINTER + // children + MEMORY_ARRAY + // Object[] children + 8; // totalCount + + /** + * The estimated number of bytes used per empty leaf page. + */ + static final int PAGE_LEAF_MEMORY = PAGE_MEMORY + // super + MEMORY_POINTER + // values + MEMORY_ARRAY; // Object[] values + + /** + * An empty object array. + */ + private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; + + /** + * Marker value for memory field, meaning that memory accounting is replaced by key count. + */ + private static final int IN_MEMORY = Integer.MIN_VALUE; + + private static final PageReference[] SINGLE_EMPTY = {PageReference.EMPTY}; + + + Page(MVMap map) { + this.map = map; + } + + Page(MVMap map, Page source) { + this(map, source.keys); + memory = source.memory; + } + + Page(MVMap map, Object[] keys) { + this.map = map; + this.keys = keys; + } + + /** + * Create a new, empty leaf page. + * + * @param map the map + * @return the new page + */ + static Page createEmptyLeaf(MVMap map) { + return createLeaf(map, EMPTY_OBJECT_ARRAY, EMPTY_OBJECT_ARRAY, PAGE_LEAF_MEMORY); + } + + /** + * Create a new, empty internal node page. + * + * @param map the map + * @return the new page + */ + static Page createEmptyNode(MVMap map) { + return createNode(map, EMPTY_OBJECT_ARRAY, SINGLE_EMPTY, 0, PAGE_NODE_MEMORY + MEMORY_POINTER + PAGE_MEMORY_CHILD); // there is always one child + } + + /** + * Create a new non-leaf page. The arrays are not cloned. + * + * @param map the map + * @param keys the keys + * @param children the child page positions + * @param totalCount the total number of keys + * @param memory the memory used in bytes + * @return the page + */ + public static Page createNode(MVMap map, Object[] keys, PageReference[] children, long totalCount, int memory) { + assert keys != null; + Page page = new NonLeaf(map, keys, children, totalCount); + page.initMemoryAccount(memory); + return page; + } + + /** + * Create a new leaf page. The arrays are not cloned. + * + * @param map the map + * @param keys the keys + * @param values the values + * @param memory the memory used in bytes + * @return the page + */ + static Page createLeaf(MVMap map, Object[] keys, Object[] values, int memory) { + assert keys != null; + Page page = new Leaf(map, keys, values); + page.initMemoryAccount(memory); + return page; + } + + private void initMemoryAccount(int memoryCount) { + if (!map.isPersistent()) { + memory = IN_MEMORY; + } else if (memoryCount == 0) { + recalculateMemory(); + } else { + addMemory(memoryCount); + assert memoryCount == getMemory(); + } + } + + /** + * Get the value for the given key, or null if not found. + * Search is done in the tree rooted at given page. + * + * @param key the key + * @param p the root page + * @return the value, or null if not found + */ + static Object get(Page p, Object key) { + while (true) { + int index = p.binarySearch(key); + if (p.isLeaf()) { + return index >= 0 ? p.getValue(index) : null; + } else if (index++ < 0) { + index = -index; + } + p = p.getChildPage(index); + } + } + + /** + * Read a page. + * + * @param buff ByteBuffer containing serialized page info + * @param pos the position + * @param map the map + * @return the page + */ + static Page read(ByteBuffer buff, long pos, MVMap map) { + boolean leaf = (DataUtils.getPageType(pos) & 1) == PAGE_TYPE_LEAF; + Page p = leaf ? new Leaf(map) : new NonLeaf(map); + p.pos = pos; + int chunkId = DataUtils.getPageChunkId(pos); + p.read(buff, chunkId); + return p; + } + + /** + * Get the id of the page's owner map + * + * @return id + */ + public final int getMapId() { + return map.getId(); + } + + /** + * Create a copy of this page with potentially different owning map. + * This is used exclusively during bulk map copying. + * Child page references for nodes are cleared (re-pointed to an empty page) + * to be filled-in later to copying procedure. This way it can be saved + * mid-process without tree integrity violation + * + * @param map new map to own resulting page + * @return the page + */ + abstract Page copy(MVMap map); + + /** + * Get the key at the given index. + * + * @param index the index + * @return the key + */ + public Object getKey(int index) { + return keys[index]; + } + + /** + * Get the child page at the given index. + * + * @param index the index + * @return the child page + */ + public abstract Page getChildPage(int index); + + /** + * Get the position of the child. + * + * @param index the index + * @return the position + */ + public abstract long getChildPagePos(int index); + + /** + * Get the value at the given index. + * + * @param index the index + * @return the value + */ + public abstract Object getValue(int index); + + /** + * Get the number of keys in this page. + * + * @return the number of keys + */ + public final int getKeyCount() { + return keys.length; + } + + /** + * Check whether this is a leaf page. + * + * @return true if it is a leaf + */ + public final boolean isLeaf() { + return getNodeType() == PAGE_TYPE_LEAF; + } + + public abstract int getNodeType(); + + /** + * Get the position of the page + * + * @return the position + */ + public final long getPos() { + return pos; + } + + @Override + public String toString() { + StringBuilder buff = new StringBuilder(); + dump(buff); + return buff.toString(); + } + + /** + * Dump debug data for this page. + * + * @param buff append buffer + */ + protected void dump(StringBuilder buff) { + buff.append("id: ").append(System.identityHashCode(this)).append('\n'); + buff.append("pos: ").append(Long.toHexString(pos)).append('\n'); + if (isSaved()) { + int chunkId = DataUtils.getPageChunkId(pos); + buff.append("chunk: ").append(Long.toHexString(chunkId)).append('\n'); + } + } + + /** + * Create a copy of this page. + * + * @return a mutable copy of this page + */ + public final Page copy() { + Page newPage = clone(); + newPage.pos = 0; + return newPage; + } + + @Override + protected final Page clone() { + Page clone; + try { + clone = (Page) super.clone(); + } catch (CloneNotSupportedException impossible) { + throw new RuntimeException(impossible); + } + return clone; + } + + /** + * Search the key in this page using a binary search. Instead of always + * starting the search in the middle, the last found index is cached. + *

+ * If the key was found, the returned value is the index in the key array. + * If not found, the returned value is negative, where -1 means the provided + * key is smaller than any keys in this page. See also Arrays.binarySearch. + * + * @param key the key + * @return the value or null + */ + int binarySearch(Object key) { + int low = 0, high = getKeyCount() - 1; + // the cached index minus one, so that + // for the first time (when cachedCompare is 0), + // the default value is used + int x = cachedCompare - 1; + if (x < 0 || x > high) { + x = high >>> 1; + } + Object[] k = keys; + while (low <= high) { + int compare = map.compare(key, k[x]); + if (compare > 0) { + low = x + 1; + } else if (compare < 0) { + high = x - 1; + } else { + cachedCompare = x + 1; + return x; + } + x = (low + high) >>> 1; + } + cachedCompare = low; + return -(low + 1); + } + + /** + * Split the page. This modifies the current page. + * + * @param at the split index + * @return the page with the entries after the split index + */ + abstract Page split(int at); + + /** + * Split the current keys array into two arrays. + * + * @param aCount size of the first array. + * @param bCount size of the second array/ + * @return the second array. + */ + final Object[] splitKeys(int aCount, int bCount) { + assert aCount + bCount <= getKeyCount(); + Object[] aKeys = createKeyStorage(aCount); + Object[] bKeys = createKeyStorage(bCount); + System.arraycopy(keys, 0, aKeys, 0, aCount); + System.arraycopy(keys, getKeyCount() - bCount, bKeys, 0, bCount); + keys = aKeys; + return bKeys; + } + + /** + * Append additional key/value mappings to this Page. + * New mappings suppose to be in correct key order. + * + * @param extraKeyCount number of mappings to be added + * @param extraKeys to be added + * @param extraValues to be added + */ + abstract void expand(int extraKeyCount, Object[] extraKeys, Object[] extraValues); + + /** + * Expand the keys array. + * + * @param extraKeyCount number of extra key entries to create + * @param extraKeys extra key values + */ + final void expandKeys(int extraKeyCount, Object[] extraKeys) { + int keyCount = getKeyCount(); + Object[] newKeys = createKeyStorage(keyCount + extraKeyCount); + System.arraycopy(keys, 0, newKeys, 0, keyCount); + System.arraycopy(extraKeys, 0, newKeys, keyCount, extraKeyCount); + keys = newKeys; + } + + /** + * Get the total number of key-value pairs, including child pages. + * + * @return the number of key-value pairs + */ + public abstract long getTotalCount(); + + /** + * Get the number of key-value pairs for a given child. + * + * @param index the child index + * @return the descendant count + */ + abstract long getCounts(int index); + + /** + * Replace the child page. + * + * @param index the index + * @param c the new child page + */ + public abstract void setChild(int index, Page c); + + /** + * Replace the key at an index in this page. + * + * @param index the index + * @param key the new key + */ + public final void setKey(int index, Object key) { + keys = keys.clone(); + if (isPersistent()) { + Object old = keys[index]; + DataType keyType = map.getKeyType(); + int mem = keyType.getMemory(key); + if (old != null) { + mem -= keyType.getMemory(old); + } + addMemory(mem); + } + keys[index] = key; + } + + /** + * Replace the value at an index in this page. + * + * @param index the index + * @param value the new value + * @return the old value + */ + public abstract Object setValue(int index, Object value); + + /** + * Insert a key-value pair into this leaf. + * + * @param index the index + * @param key the key + * @param value the value + */ + public abstract void insertLeaf(int index, Object key, Object value); + + /** + * Insert a child page into this node. + * + * @param index the index + * @param key the key + * @param childPage the child page + */ + public abstract void insertNode(int index, Object key, Page childPage); + + /** + * Insert a key into the key array + * + * @param index index to insert at + * @param key the key value + */ + final void insertKey(int index, Object key) { + int keyCount = getKeyCount(); + assert index <= keyCount : index + " > " + keyCount; + Object[] newKeys = createKeyStorage(keyCount + 1); + DataUtils.copyWithGap(keys, newKeys, keyCount, index); + keys = newKeys; + + keys[index] = key; + + if (isPersistent()) { + addMemory(MEMORY_POINTER + map.getKeyType().getMemory(key)); + } + } + + /** + * Remove the key and value (or child) at the given index. + * + * @param index the index + */ + public void remove(int index) { + int keyCount = getKeyCount(); + org.dizitart.no2.mvstore.compat.v1.mvstore.type.DataType keyType = map.getKeyType(); + if (index == keyCount) { + --index; + } + if (isPersistent()) { + Object old = getKey(index); + addMemory(-MEMORY_POINTER - keyType.getMemory(old)); + } + Object[] newKeys = createKeyStorage(keyCount - 1); + DataUtils.copyExcept(keys, newKeys, keyCount, index); + keys = newKeys; + } + + /** + * Read the page from the buffer. + * + * @param buff the buffer + * @param chunkId the chunk id + */ + private void read(ByteBuffer buff, int chunkId) { + // size of int + short + varint, since we've read page length, check and + // mapId already + int pageLength = buff.remaining() + 10; + int len = DataUtils.readVarInt(buff); + keys = createKeyStorage(len); + int type = buff.get(); + if (isLeaf() != ((type & 1) == PAGE_TYPE_LEAF)) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT, "File corrupted in chunk {0}, expected node type {1}, got {2}", chunkId, isLeaf() ? "0" : "1", type); + } + if (!isLeaf()) { + readPayLoad(buff); + } + boolean compressed = (type & DataUtils.PAGE_COMPRESSED) != 0; + if (compressed) { + Compressor compressor; + if ((type & DataUtils.PAGE_COMPRESSED_HIGH) == DataUtils.PAGE_COMPRESSED_HIGH) { + compressor = map.getStore().getCompressorHigh(); + } else { + compressor = map.getStore().getCompressorFast(); + } + int lenAdd = DataUtils.readVarInt(buff); + int compLen = buff.remaining(); + byte[] comp = Utils.newBytes(compLen); + buff.get(comp); + int l = compLen + lenAdd; + buff = ByteBuffer.allocate(l); + compressor.expand(comp, 0, compLen, buff.array(), buff.arrayOffset(), l); + } + map.getKeyType().read(buff, keys, len, true); + if (isLeaf()) { + readPayLoad(buff); + } + diskSpaceUsed = pageLength; + recalculateMemory(); + } + + /** + * Read the page payload from the buffer. + * + * @param buff the buffer + */ + protected abstract void readPayLoad(ByteBuffer buff); + + public final boolean isSaved() { + return DataUtils.isPageSaved(pos); + } + + public final boolean isRemoved() { + return DataUtils.isPageRemoved(pos); + } + + /** + * Mark this page as removed "in memory". That means that only adjustment of + * "unsaved memory" amount is required. On the other hand, if page was + * persisted, it's removal should be reflected in occupancy of the + * containing chunk. + * + * @return true if it was marked by this call or has been marked already, + * false if page has been saved already. + */ + private boolean markAsRemoved() { + assert getTotalCount() > 0 : this; + long pagePos; + do { + pagePos = pos; + if (DataUtils.isPageSaved(pagePos)) { + return false; + } + assert !DataUtils.isPageRemoved(pagePos); + } while (!posUpdater.compareAndSet(this, 0L, 1L)); + return true; + } + + /** + * Store the page and update the position. + * + * @param chunk the chunk + * @param buff the target buffer + * @return the position of the buffer just after the type + */ + protected final int write(Chunk chunk, WriteBuffer buff) { + int start = buff.position(); + int len = getKeyCount(); + int type = isLeaf() ? PAGE_TYPE_LEAF : DataUtils.PAGE_TYPE_NODE; + buff.putInt(0).putShort((byte) 0).putVarInt(map.getId()).putVarInt(len); + int typePos = buff.position(); + buff.put((byte) type); + writeChildren(buff, true); + int compressStart = buff.position(); + map.getKeyType().write(buff, keys, len, true); + writeValues(buff); + MVStore store = map.getStore(); + int expLen = buff.position() - compressStart; + if (expLen > 16) { + int compressionLevel = store.getCompressionLevel(); + if (compressionLevel > 0) { + Compressor compressor; + int compressType; + if (compressionLevel == 1) { + compressor = map.getStore().getCompressorFast(); + compressType = DataUtils.PAGE_COMPRESSED; + } else { + compressor = map.getStore().getCompressorHigh(); + compressType = DataUtils.PAGE_COMPRESSED_HIGH; + } + byte[] exp = new byte[expLen]; + buff.position(compressStart).get(exp); + byte[] comp = new byte[expLen * 2]; + int compLen = compressor.compress(exp, expLen, comp, 0); + int plus = DataUtils.getVarIntLen(compLen - expLen); + if (compLen + plus < expLen) { + buff.position(typePos).put((byte) (type + compressType)); + buff.position(compressStart).putVarInt(expLen - compLen).put(comp, 0, compLen); + } + } + } + int pageLength = buff.position() - start; + int chunkId = chunk.id; + int check = DataUtils.getCheckValue(chunkId) ^ DataUtils.getCheckValue(start) ^ DataUtils.getCheckValue(pageLength); + buff.putInt(start, pageLength).putShort(start + 4, (short) check); + if (isSaved()) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_INTERNAL, "Page already stored"); + } + long pagePos = DataUtils.getPagePos(chunkId, start, pageLength, type); + boolean isDeleted = isRemoved(); + while (!posUpdater.compareAndSet(this, isDeleted ? 1L : 0L, pagePos)) { + isDeleted = isRemoved(); + } + store.cachePage(this); + if (type == DataUtils.PAGE_TYPE_NODE) { + // cache again - this will make sure nodes stays in the cache + // for a longer time + store.cachePage(this); + } + int pageLengthEncoded = DataUtils.getPageMaxLength(pos); + boolean singleWriter = map.isSingleWriter(); + chunk.accountForWrittenPage(pageLengthEncoded, singleWriter); + if (isDeleted) { + store.accountForRemovedPage(pagePos, chunk.version + 1, singleWriter); + } + diskSpaceUsed = pageLengthEncoded != DataUtils.PAGE_LARGE ? pageLengthEncoded : pageLength; + return typePos + 1; + } + + /** + * Write values that the buffer contains to the buff. + * + * @param buff the target buffer + */ + protected abstract void writeValues(WriteBuffer buff); + + /** + * Write page children to the buff. + * + * @param buff the target buffer + * @param withCounts true if the descendant counts should be written + */ + protected abstract void writeChildren(WriteBuffer buff, boolean withCounts); + + /** + * Store this page and all children that are changed, in reverse order, and + * update the position and the children. + * + * @param chunk the chunk + * @param buff the target buffer + */ + abstract void writeUnsavedRecursive(Chunk chunk, WriteBuffer buff); + + /** + * Unlink the children recursively after all data is written. + */ + abstract void writeEnd(); + + public abstract int getRawChildPageCount(); + + @Override + public final boolean equals(Object other) { + return other == this || other instanceof Page && isSaved() && ((Page) other).pos == pos; + } + + @Override + public final int hashCode() { + return isSaved() ? (int) (pos | (pos >>> 32)) : super.hashCode(); + } + + protected final boolean isPersistent() { + return memory != IN_MEMORY; + } + + public final int getMemory() { + if (isPersistent()) { +// assert memory == calculateMemory() : +// "Memory calculation error " + memory + " != " + calculateMemory(); + return memory; + } + return 0; + } + + /** + * Amount of used disk space in persistent case including child pages. + * + * @return amount of used disk space in persistent case + */ + public long getDiskSpaceUsed() { + long r = 0; + if (isPersistent()) { + r += diskSpaceUsed; + if (!isLeaf()) { + for (int i = 0; i < getRawChildPageCount(); i++) { + long pos = getChildPagePos(i); + if (pos != 0) { + r += getChildPage(i).getDiskSpaceUsed(); + } + } + } + } + return r; + } + + /** + * Increase estimated memory used in persistent case. + * + * @param mem additional memory size. + */ + final void addMemory(int mem) { + memory += mem; + } + + /** + * Recalculate estimated memory used in persistent case. + */ + final void recalculateMemory() { + assert isPersistent(); + memory = calculateMemory(); + } + + /** + * Calculate estimated memory used in persistent case. + * + * @return memory in bytes + */ + protected int calculateMemory() { + int keyCount = getKeyCount(); + int mem = keyCount * MEMORY_POINTER; + DataType keyType = map.getKeyType(); + for (int i = 0; i < keyCount; i++) { + mem += keyType.getMemory(keys[i]); + } + return mem; + } + + public boolean isComplete() { + return true; + } + + /** + * Called when done with copying page. + */ + public void setComplete() { + } + + /** + * Make accounting changes (chunk occupancy or "unsaved" RAM), related to + * this page removal. + * + * @param version at which page was removed + * @return amount (negative), by which "unsaved memory" should be adjusted, + * if page is unsaved one, and 0 for page that was already saved, or + * in case of non-persistent map + */ + public final int removePage(long version) { + if (isPersistent() && getTotalCount() > 0) { + MVStore store = map.store; + if (!markAsRemoved()) { // only if it has been saved already + long pagePos = pos; + store.accountForRemovedPage(pagePos, version, map.isSingleWriter()); + } else { + return -memory; + } + } + return 0; + } + + /** + * Extend path from a given CursorPos chain to "prepend point" in a B-tree, rooted at this Page. + * + * @param cursorPos presumably pointing to this Page (null if real root), to build upon + * @return new head of the CursorPos chain + */ + public abstract CursorPos getPrependCursorPos(CursorPos cursorPos); + + /** + * Extend path from a given CursorPos chain to "append point" in a B-tree, rooted at this Page. + * + * @param cursorPos presumably pointing to this Page (null if real root), to build upon + * @return new head of the CursorPos chain + */ + public abstract CursorPos getAppendCursorPos(CursorPos cursorPos); + + /** + * Remove all page data recursively. + * + * @param version at which page got removed + * @return adjustment for "unsaved memory" amount + */ + public abstract int removeAllRecursive(long version); + + /** + * Create array for keys storage. + * + * @param size number of entries + * @return values array + */ + private Object[] createKeyStorage(int size) { + return new Object[size]; + } + + /** + * Create array for values storage. + * + * @param size number of entries + * @return values array + */ + final Object[] createValueStorage(int size) { + return new Object[size]; + } + + /** + * A pointer to a page, either in-memory or using a page position. + */ + public static final class PageReference { + + /** + * Singleton object used when arrays of PageReference have not yet been filled. + */ + public static final PageReference EMPTY = new PageReference(null, 0, 0); + + /** + * The position, if known, or 0. + */ + private long pos; + + /** + * The page, if in memory, or null. + */ + private Page page; + + /** + * The descendant count for this child page. + */ + final long count; + + public PageReference(Page page) { + this(page, page.getPos(), page.getTotalCount()); + } + + PageReference(long pos, long count) { + this(null, pos, count); + assert DataUtils.isPageSaved(pos); + } + + private PageReference(Page page, long pos, long count) { + this.page = page; + this.pos = pos; + this.count = count; + } + + public Page getPage() { + return page; + } + + /** + * Clear if necessary, reference to the actual child Page object, + * so it can be garbage collected if not actively used elsewhere. + * Reference is cleared only if corresponding page was already saved on a disk. + */ + void clearPageReference() { + if (page != null) { + page.writeEnd(); + assert page.isSaved() || !page.isComplete(); + if (page.isSaved()) { + assert pos == page.getPos(); + assert count == page.getTotalCount() : count + " != " + page.getTotalCount(); + page = null; + } + } + } + + long getPos() { + return pos; + } + + /** + * Re-acquire position from in-memory page. + */ + void resetPos() { + Page p = page; + if (p != null && p.isSaved()) { + pos = p.getPos(); + assert count == p.getTotalCount(); + } + } + + @Override + public String toString() { + return "Cnt:" + count + ", pos:" + (pos == 0 ? "0" : DataUtils.getPageChunkId(pos) + "-" + DataUtils.getPageOffset(pos) + ":" + DataUtils.getPageMaxLength(pos)) + ((page == null ? DataUtils.getPageType(pos) == 0 : page.isLeaf()) ? " leaf" : " node") + ", page:{" + page + "}"; + } + } + + + private static class NonLeaf extends Page { + /** + * The child page references. + */ + private PageReference[] children; + + /** + * The total entry count of this page and all children. + */ + private long totalCount; + + NonLeaf(MVMap map) { + super(map); + } + + NonLeaf(MVMap map, NonLeaf source, PageReference[] children, long totalCount) { + super(map, source); + this.children = children; + this.totalCount = totalCount; + } + + NonLeaf(MVMap map, Object[] keys, PageReference[] children, long totalCount) { + super(map, keys); + this.children = children; + this.totalCount = totalCount; + } + + @Override + public int getNodeType() { + return DataUtils.PAGE_TYPE_NODE; + } + + @Override + public Page copy(MVMap map) { + return new IncompleteNonLeaf(map, this); + } + + @Override + public Page getChildPage(int index) { + PageReference ref = children[index]; + Page page = ref.getPage(); + if (page == null) { + page = map.readPage(ref.getPos()); + assert ref.getPos() == page.getPos(); + assert ref.count == page.getTotalCount(); + } + return page; + } + + @Override + public long getChildPagePos(int index) { + return children[index].getPos(); + } + + @Override + public Object getValue(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public Page split(int at) { + assert !isSaved(); + int b = getKeyCount() - at; + Object[] bKeys = splitKeys(at, b - 1); + PageReference[] aChildren = new PageReference[at + 1]; + PageReference[] bChildren = new PageReference[b]; + System.arraycopy(children, 0, aChildren, 0, at + 1); + System.arraycopy(children, at + 1, bChildren, 0, b); + children = aChildren; + + long t = 0; + for (PageReference x : aChildren) { + t += x.count; + } + totalCount = t; + t = 0; + for (PageReference x : bChildren) { + t += x.count; + } + Page newPage = createNode(map, bKeys, bChildren, t, 0); + if (isPersistent()) { + recalculateMemory(); + } + return newPage; + } + + @Override + public void expand(int keyCount, Object[] extraKeys, Object[] extraValues) { + throw new UnsupportedOperationException(); + } + + @Override + public long getTotalCount() { + assert !isComplete() || totalCount == calculateTotalCount() : "Total count: " + totalCount + " != " + calculateTotalCount(); + return totalCount; + } + + private long calculateTotalCount() { + long check = 0; + int keyCount = getKeyCount(); + for (int i = 0; i <= keyCount; i++) { + check += children[i].count; + } + return check; + } + + void recalculateTotalCount() { + totalCount = calculateTotalCount(); + } + + @Override + long getCounts(int index) { + return children[index].count; + } + + @Override + public void setChild(int index, Page c) { + assert c != null; + PageReference child = children[index]; + if (c != child.getPage() || c.getPos() != child.getPos()) { + totalCount += c.getTotalCount() - child.count; + children = children.clone(); + children[index] = new PageReference(c); + } + } + + @Override + public Object setValue(int index, Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public void insertLeaf(int index, Object key, Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public void insertNode(int index, Object key, Page childPage) { + int childCount = getRawChildPageCount(); + insertKey(index, key); + + PageReference[] newChildren = new PageReference[childCount + 1]; + DataUtils.copyWithGap(children, newChildren, childCount, index); + children = newChildren; + children[index] = new PageReference(childPage); + + totalCount += childPage.getTotalCount(); + if (isPersistent()) { + addMemory(MEMORY_POINTER + PAGE_MEMORY_CHILD); + } + } + + @Override + public void remove(int index) { + int childCount = getRawChildPageCount(); + super.remove(index); + if (isPersistent()) { + addMemory(-MEMORY_POINTER - PAGE_MEMORY_CHILD); + } + totalCount -= children[index].count; + PageReference[] newChildren = new PageReference[childCount - 1]; + DataUtils.copyExcept(children, newChildren, childCount, index); + children = newChildren; + } + + @Override + public int removeAllRecursive(long version) { + int unsavedMemory = removePage(version); + if (isPersistent()) { + for (int i = 0, size = map.getChildPageCount(this); i < size; i++) { + PageReference ref = children[i]; + Page page = ref.getPage(); + if (page != null) { + unsavedMemory += page.removeAllRecursive(version); + } else { + long pagePos = ref.getPos(); + assert DataUtils.isPageSaved(pagePos); + if (DataUtils.isLeafPosition(pagePos)) { + map.store.accountForRemovedPage(pagePos, version, map.isSingleWriter()); + } else { + unsavedMemory += map.readPage(pagePos).removeAllRecursive(version); + } + } + } + } + return unsavedMemory; + } + + @Override + public CursorPos getPrependCursorPos(CursorPos cursorPos) { + Page childPage = getChildPage(0); + return childPage.getPrependCursorPos(new CursorPos(this, 0, cursorPos)); + } + + @Override + public CursorPos getAppendCursorPos(CursorPos cursorPos) { + int keyCount = getKeyCount(); + Page childPage = getChildPage(keyCount); + return childPage.getAppendCursorPos(new CursorPos(this, keyCount, cursorPos)); + } + + @Override + protected void readPayLoad(ByteBuffer buff) { + int keyCount = getKeyCount(); + children = new PageReference[keyCount + 1]; + long[] p = new long[keyCount + 1]; + for (int i = 0; i <= keyCount; i++) { + p[i] = buff.getLong(); + } + long total = 0; + for (int i = 0; i <= keyCount; i++) { + long s = DataUtils.readVarLong(buff); + long position = p[i]; + assert position == 0 ? s == 0 : s >= 0; + total += s; + children[i] = position == 0 ? PageReference.EMPTY : new PageReference(position, s); + } + totalCount = total; + } + + @Override + protected void writeValues(WriteBuffer buff) { + } + + @Override + protected void writeChildren(WriteBuffer buff, boolean withCounts) { + int keyCount = getKeyCount(); + for (int i = 0; i <= keyCount; i++) { + buff.putLong(children[i].getPos()); + } + if (withCounts) { + for (int i = 0; i <= keyCount; i++) { + buff.putVarLong(children[i].count); + } + } + } + + @Override + void writeUnsavedRecursive(Chunk chunk, WriteBuffer buff) { + if (!isSaved()) { + int patch = write(chunk, buff); + writeChildrenRecursive(chunk, buff); + int old = buff.position(); + buff.position(patch); + writeChildren(buff, false); + buff.position(old); + } + } + + void writeChildrenRecursive(Chunk chunk, WriteBuffer buff) { + int len = getRawChildPageCount(); + for (int i = 0; i < len; i++) { + PageReference ref = children[i]; + Page p = ref.getPage(); + if (p != null) { + p.writeUnsavedRecursive(chunk, buff); + ref.resetPos(); + } + } + } + + @Override + void writeEnd() { + int len = getRawChildPageCount(); + for (int i = 0; i < len; i++) { + children[i].clearPageReference(); + } + } + + @Override + public int getRawChildPageCount() { + return getKeyCount() + 1; + } + + @Override + protected int calculateMemory() { + return super.calculateMemory() + PAGE_NODE_MEMORY + getRawChildPageCount() * (MEMORY_POINTER + PAGE_MEMORY_CHILD); + } + + @Override + public void dump(StringBuilder buff) { + super.dump(buff); + int keyCount = getKeyCount(); + for (int i = 0; i <= keyCount; i++) { + if (i > 0) { + buff.append(" "); + } + buff.append("[").append(Long.toHexString(children[i].getPos())).append("]"); + if (i < keyCount) { + buff.append(" ").append(getKey(i)); + } + } + } + } + + + private static class IncompleteNonLeaf extends NonLeaf { + + private boolean complete; + + IncompleteNonLeaf(MVMap map, NonLeaf source) { + super(map, source, constructEmptyPageRefs(source.getRawChildPageCount()), source.getTotalCount()); + } + + private static PageReference[] constructEmptyPageRefs(int size) { + // replace child pages with empty pages + PageReference[] children = new PageReference[size]; + Arrays.fill(children, PageReference.EMPTY); + return children; + } + + @Override + void writeUnsavedRecursive(Chunk chunk, WriteBuffer buff) { + if (complete) { + super.writeUnsavedRecursive(chunk, buff); + } else if (!isSaved()) { + writeChildrenRecursive(chunk, buff); + } + } + + @Override + public boolean isComplete() { + return complete; + } + + @Override + public void setComplete() { + recalculateTotalCount(); + complete = true; + } + + @Override + public void dump(StringBuilder buff) { + super.dump(buff); + buff.append(", complete:").append(complete); + } + + } + + + private static class Leaf extends Page { + /** + * The storage for values. + */ + private Object[] values; + + Leaf(MVMap map) { + super(map); + } + + private Leaf(MVMap map, Leaf source) { + super(map, source); + this.values = source.values; + } + + Leaf(MVMap map, Object[] keys, Object[] values) { + super(map, keys); + this.values = values; + } + + @Override + public int getNodeType() { + return PAGE_TYPE_LEAF; + } + + @Override + public Page copy(MVMap map) { + return new Leaf(map, this); + } + + @Override + public Page getChildPage(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public long getChildPagePos(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public Object getValue(int index) { + return values[index]; + } + + @Override + public Page split(int at) { + assert !isSaved(); + int b = getKeyCount() - at; + Object[] bKeys = splitKeys(at, b); + Object[] bValues = createValueStorage(b); + if (values != null) { + Object[] aValues = createValueStorage(at); + System.arraycopy(values, 0, aValues, 0, at); + System.arraycopy(values, at, bValues, 0, b); + values = aValues; + } + Page newPage = createLeaf(map, bKeys, bValues, 0); + if (isPersistent()) { + recalculateMemory(); + } + return newPage; + } + + @Override + public void expand(int extraKeyCount, Object[] extraKeys, Object[] extraValues) { + int keyCount = getKeyCount(); + expandKeys(extraKeyCount, extraKeys); + if (values != null) { + Object[] newValues = createValueStorage(keyCount + extraKeyCount); + System.arraycopy(values, 0, newValues, 0, keyCount); + System.arraycopy(extraValues, 0, newValues, keyCount, extraKeyCount); + values = newValues; + } + if (isPersistent()) { + recalculateMemory(); + } + } + + @Override + public long getTotalCount() { + return getKeyCount(); + } + + @Override + long getCounts(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public void setChild(int index, Page c) { + throw new UnsupportedOperationException(); + } + + @Override + public Object setValue(int index, Object value) { + DataType valueType = map.getValueType(); + values = values.clone(); + Object old = setValueInternal(index, value); + if (isPersistent()) { + addMemory(valueType.getMemory(value) - valueType.getMemory(old)); + } + return old; + } + + private Object setValueInternal(int index, Object value) { + Object old = values[index]; + values[index] = value; + return old; + } + + @Override + public void insertLeaf(int index, Object key, Object value) { + int keyCount = getKeyCount(); + insertKey(index, key); + + if (values != null) { + Object[] newValues = createValueStorage(keyCount + 1); + DataUtils.copyWithGap(values, newValues, keyCount, index); + values = newValues; + setValueInternal(index, value); + if (isPersistent()) { + addMemory(MEMORY_POINTER + map.getValueType().getMemory(value)); + } + } + } + + @Override + public void insertNode(int index, Object key, Page childPage) { + throw new UnsupportedOperationException(); + } + + @Override + public void remove(int index) { + int keyCount = getKeyCount(); + super.remove(index); + if (values != null) { + if (isPersistent()) { + Object old = getValue(index); + addMemory(-MEMORY_POINTER - map.getValueType().getMemory(old)); + } + Object[] newValues = createValueStorage(keyCount - 1); + DataUtils.copyExcept(values, newValues, keyCount, index); + values = newValues; + } + } + + @Override + public int removeAllRecursive(long version) { + return removePage(version); + } + + @Override + public CursorPos getPrependCursorPos(CursorPos cursorPos) { + return new CursorPos(this, -1, cursorPos); + } + + @Override + public CursorPos getAppendCursorPos(CursorPos cursorPos) { + int keyCount = getKeyCount(); + return new CursorPos(this, -keyCount - 1, cursorPos); + } + + @Override + protected void readPayLoad(ByteBuffer buff) { + int keyCount = getKeyCount(); + values = createValueStorage(keyCount); + map.getValueType().read(buff, values, getKeyCount(), false); + } + + @Override + protected void writeValues(WriteBuffer buff) { + map.getValueType().write(buff, values, getKeyCount(), false); + } + + @Override + protected void writeChildren(WriteBuffer buff, boolean withCounts) { + } + + @Override + void writeUnsavedRecursive(Chunk chunk, WriteBuffer buff) { + if (!isSaved()) { + write(chunk, buff); + } + } + + @Override + void writeEnd() { + } + + @Override + public int getRawChildPageCount() { + return 0; + } + + @Override + protected int calculateMemory() { + int keyCount = getKeyCount(); + int mem = super.calculateMemory() + PAGE_LEAF_MEMORY + keyCount * MEMORY_POINTER; + DataType valueType = map.getValueType(); + for (int i = 0; i < keyCount; i++) { + mem += valueType.getMemory(values[i]); + } + return mem; + } + + @Override + public void dump(StringBuilder buff) { + super.dump(buff); + int keyCount = getKeyCount(); + for (int i = 0; i < keyCount; i++) { + if (i > 0) { + buff.append(" "); + } + buff.append(getKey(i)); + if (values != null) { + buff.append(':'); + buff.append(getValue(i)); + } + } + } + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/RootReference.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/RootReference.java new file mode 100644 index 000000000..e7200bd6f --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/RootReference.java @@ -0,0 +1,263 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +/** + * Class RootReference is an immutable structure to represent state of the MVMap as a whole + * (not related to a particular B-Tree node). + * Single structure would allow for non-blocking atomic state change. + * The most important part of it is a reference to the root node. + * + * @author Andrei Tokar + */ +public final class RootReference +{ + /** + * The root page. + */ + public final Page root; + /** + * The version used for writing. + */ + public final long version; + /** + * Counter of reenterant locks. + */ + private final byte holdCount; + /** + * Lock owner thread id. + */ + private final long ownerId; + /** + * Reference to the previous root in the chain. + * That is the last root of the previous version, which had any data changes. + * Versions without any data changes are dropped from the chain, as it built. + */ + volatile RootReference previous; + /** + * Counter for successful root updates. + */ + final long updateCounter; + /** + * Counter for attempted root updates. + */ + final long updateAttemptCounter; + /** + * Size of the occupied part of the append buffer. + */ + private final byte appendCounter; + + + // This one is used to set root initially and for r/o snapshots + RootReference(Page root, long version) { + this.root = root; + this.version = version; + this.previous = null; + this.updateCounter = 1; + this.updateAttemptCounter = 1; + this.holdCount = 0; + this.ownerId = 0; + this.appendCounter = 0; + } + + private RootReference(RootReference r, Page root, long updateAttemptCounter) { + this.root = root; + this.version = r.version; + this.previous = r.previous; + this.updateCounter = r.updateCounter + 1; + this.updateAttemptCounter = r.updateAttemptCounter + updateAttemptCounter; + this.holdCount = 0; + this.ownerId = 0; + this.appendCounter = r.appendCounter; + } + + // This one is used for locking + private RootReference(RootReference r, int attempt) { + this.root = r.root; + this.version = r.version; + this.previous = r.previous; + this.updateCounter = r.updateCounter + 1; + this.updateAttemptCounter = r.updateAttemptCounter + attempt; + assert r.holdCount == 0 || r.ownerId == Thread.currentThread().getId() // + : Thread.currentThread().getId() + " " + r; + this.holdCount = (byte)(r.holdCount + 1); + this.ownerId = Thread.currentThread().getId(); + this.appendCounter = r.appendCounter; + } + + // This one is used for unlocking + private RootReference(RootReference r, Page root, boolean keepLocked, int appendCounter) { + this.root = root; + this.version = r.version; + this.previous = r.previous; + this.updateCounter = r.updateCounter; + this.updateAttemptCounter = r.updateAttemptCounter; + assert r.holdCount > 0 && r.ownerId == Thread.currentThread().getId() // + : Thread.currentThread().getId() + " " + r; + this.holdCount = (byte)(r.holdCount - (keepLocked ? 0 : 1)); + this.ownerId = this.holdCount == 0 ? 0 : Thread.currentThread().getId(); + this.appendCounter = (byte) appendCounter; + } + + // This one is used for version change + private RootReference(RootReference r, long version, int attempt) { + RootReference previous = r; + RootReference tmp; + while ((tmp = previous.previous) != null && tmp.root == r.root) { + previous = tmp; + } + this.root = r.root; + this.version = version; + this.previous = previous; + this.updateCounter = r.updateCounter + 1; + this.updateAttemptCounter = r.updateAttemptCounter + attempt; + this.holdCount = r.holdCount == 0 ? 0 : (byte)(r.holdCount - 1); + this.ownerId = this.holdCount == 0 ? 0 : r.ownerId; + assert r.appendCounter == 0; + this.appendCounter = 0; + } + + /** + * Try to unlock. + * + * @param newRootPage the new root page + * @param attemptCounter the number of attempts so far + * @return the new, unlocked, root reference, or null if not successful + */ + RootReference updateRootPage(Page newRootPage, long attemptCounter) { + if (holdCount == 0) { + RootReference updatedRootReference = new RootReference(this, newRootPage, attemptCounter); + if (root.map.compareAndSetRoot(this, updatedRootReference)) { + return updatedRootReference; + } + } + return null; + } + + /** + * Try to lock. + * + * @param attemptCounter the number of attempts so far + * @return the new, locked, root reference, or null if not successful + */ + RootReference tryLock(int attemptCounter) { + if (holdCount == 0 || ownerId == Thread.currentThread().getId()) { + RootReference lockedRootReference = new RootReference(this, attemptCounter); + if (root.map.compareAndSetRoot(this, lockedRootReference)) { + return lockedRootReference; + } + } + return null; + } + + /** + * Try to unlock, and if successful update the version + * + * @param version the version + * @param attempt the number of attempts so far + * @return the new, unlocked and updated, root reference, or null if not successful + */ + RootReference tryUnlockAndUpdateVersion(long version, int attempt) { + if (holdCount == 0 || ownerId == Thread.currentThread().getId()) { + RootReference updatedRootReference = new RootReference(this, version, attempt); + if (root.map.compareAndSetRoot(this, updatedRootReference)) { + return updatedRootReference; + } + } + return null; + } + + /** + * Update the page, possibly keeping it locked. + * + * @param page the page + * @param keepLocked whether to keep it locked + * @param appendCounter the number of attempts so far + * @return the new root reference, or null if not successful + */ + RootReference updatePageAndLockedStatus(Page page, boolean keepLocked, int appendCounter) { + assert isLockedByCurrentThread() : this; + RootReference updatedRootReference = new RootReference(this, page, keepLocked, appendCounter); + if (root.map.compareAndSetRoot(this, updatedRootReference)) { + return updatedRootReference; + } + return null; + } + + /** + * Removed old versions that are not longer used. + * + * @param oldestVersionToKeep the oldest version that needs to be retained + */ + void removeUnusedOldVersions(long oldestVersionToKeep) { + // We need to keep at least one previous version (if any) here, + // because in order to retain whole history of some version + // we really need last root of the previous version. + // Root labeled with version "X" is the LAST known root for that version + // and therefore the FIRST known root for the version "X+1" + for(RootReference rootRef = this; rootRef != null; rootRef = rootRef.previous) { + if (rootRef.version < oldestVersionToKeep) { + RootReference previous; + assert (previous = rootRef.previous) == null || previous.getAppendCounter() == 0 // + : oldestVersionToKeep + " " + rootRef.previous; + rootRef.previous = null; + } + } + } + + boolean isLocked() { + return holdCount != 0; + } + + public boolean isLockedByCurrentThread() { + return holdCount != 0 && ownerId == Thread.currentThread().getId(); + } + + long getVersion() { + RootReference prev = previous; + return prev == null || prev.root != root || + prev.appendCounter != appendCounter ? + version : prev.version; + } + + /** + * Does the root have changes since the specified version? + * + * @param version to check against + * @return true if this root has unsaved changes + */ + boolean hasChangesSince(long version) { + return (root.isSaved() ? getAppendCounter() > 0 : getTotalCount() > 0) || getVersion() > version; + } + + int getAppendCounter() { + return appendCounter & 0xff; + } + + /** + * Whether flushing is needed. + * + * @return true if yes + */ + public boolean needFlush() { + return appendCounter != 0; + } + + public long getTotalCount() { + return root.getTotalCount() + getAppendCounter(); + } + + @Override + public String toString() { + return "RootReference(" + System.identityHashCode(root) + + ", v=" + version + + ", owner=" + ownerId + (ownerId == Thread.currentThread().getId() ? "(current)" : "") + + ", holdCnt=" + holdCount + + ", keys=" + root.getTotalCount() + + ", append=" + getAppendCounter() + + ")"; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/SysProperties.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/SysProperties.java new file mode 100644 index 000000000..20aefda17 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/SysProperties.java @@ -0,0 +1,170 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.h2.util.MathUtils; +import org.h2.util.Utils; + +import java.io.File; + +public class SysProperties { + public static final String H2_SCRIPT_DIRECTORY = "h2.scriptDirectory"; + public static final String H2_BROWSER = "h2.browser"; + public static final String FILE_SEPARATOR; + public static final String LINE_SEPARATOR; + public static final String USER_HOME; + public static final boolean PREVIEW; + public static final String ALLOWED_CLASSES; + public static final boolean ENABLE_ANONYMOUS_TLS; + public static final String BIND_ADDRESS; + public static final boolean CHECK; + public static final String CLIENT_TRACE_DIRECTORY; + public static final int COLLATOR_CACHE_SIZE; + public static final int CONSOLE_MAX_TABLES_LIST_INDEXES; + public static final int CONSOLE_MAX_TABLES_LIST_COLUMNS; + public static final int CONSOLE_MAX_PROCEDURES_LIST_COLUMNS; + public static final boolean CONSOLE_STREAM; + public static final int CONSOLE_TIMEOUT; + public static final int DATASOURCE_TRACE_LEVEL; + public static final int DELAY_WRONG_PASSWORD_MIN; + public static final int DELAY_WRONG_PASSWORD_MAX; + public static final boolean JAVA_SYSTEM_COMPILER; + public static boolean lobCloseBetweenReads; + public static final int LOB_FILES_PER_DIRECTORY; + public static final int LOB_CLIENT_MAX_SIZE_MEMORY; + public static final int MAX_FILE_RETRY; + public static final int MAX_RECONNECT; + public static final int MAX_MEMORY_ROWS; + public static final long MAX_TRACE_DATA_LENGTH; + public static final boolean MODIFY_ON_WRITE; + public static final boolean NIO_LOAD_MAPPED; + public static final boolean NIO_CLEANER_HACK; + public static final boolean OBJECT_CACHE; + public static final int OBJECT_CACHE_MAX_PER_ELEMENT_SIZE; + public static final int OBJECT_CACHE_SIZE; + public static final boolean OLD_RESULT_SET_GET_OBJECT; + public static final boolean BIG_DECIMAL_IS_DECIMAL; + public static final boolean RETURN_OFFSET_DATE_TIME; + public static final String PG_DEFAULT_CLIENT_ENCODING; + public static final String PREFIX_TEMP_FILE; + public static boolean FORCE_AUTOCOMMIT_OFF_ON_COMMIT; + public static final int SERVER_CACHED_OBJECTS; + public static final int SERVER_RESULT_SET_FETCH_SIZE; + public static final int SOCKET_CONNECT_RETRY; + public static final int SOCKET_CONNECT_TIMEOUT; + public static final boolean SORT_BINARY_UNSIGNED; + public static final boolean SORT_UUID_UNSIGNED; + public static final boolean SORT_NULLS_HIGH; + public static final long SPLIT_FILE_SIZE_SHIFT; + public static final String SYNC_METHOD; + public static final boolean TRACE_IO; + public static final boolean THREAD_DEADLOCK_DETECTOR; + public static final boolean IMPLICIT_RELATIVE_PATH; + public static final String URL_MAP; + public static final boolean USE_THREAD_CONTEXT_CLASS_LOADER; + public static boolean serializeJavaObject; + public static final String JAVA_OBJECT_SERIALIZER; + public static final String CUSTOM_DATA_TYPES_HANDLER; + public static final String AUTH_CONFIG_FILE; + private static final String H2_BASE_DIR = "h2.baseDir"; + + private SysProperties() { + } + + public static void setBaseDir(String var0) { + if (!var0.endsWith("/")) { + var0 = var0 + "/"; + } + + System.setProperty("h2.baseDir", var0); + } + + public static String getBaseDir() { + return Utils.getProperty("h2.baseDir", (String)null); + } + + public static String getScriptDirectory() { + return Utils.getProperty("h2.scriptDirectory", ""); + } + + private static int getAutoScaledForMemoryProperty(String var0, int var1) { + String var2 = Utils.getProperty(var0, (String)null); + if (var2 != null) { + try { + return Integer.decode(var2); + } catch (NumberFormatException var4) { + } + } + + return Utils.scaleForAvailableMemory(var1); + } + + static { + FILE_SEPARATOR = File.separator; + LINE_SEPARATOR = System.lineSeparator(); + USER_HOME = Utils.getProperty("user.home", ""); + PREVIEW = Utils.getProperty("h2.preview", false); + ALLOWED_CLASSES = Utils.getProperty("h2.allowedClasses", "*"); + ENABLE_ANONYMOUS_TLS = Utils.getProperty("h2.enableAnonymousTLS", true); + BIND_ADDRESS = Utils.getProperty("h2.bindAddress", (String)null); + CHECK = Utils.getProperty("h2.check", !"0.9".equals(Utils.getProperty("java.specification.version", (String)null))); + CLIENT_TRACE_DIRECTORY = Utils.getProperty("h2.clientTraceDirectory", "trace.db/"); + COLLATOR_CACHE_SIZE = Utils.getProperty("h2.collatorCacheSize", 32000); + CONSOLE_MAX_TABLES_LIST_INDEXES = Utils.getProperty("h2.consoleTableIndexes", 100); + CONSOLE_MAX_TABLES_LIST_COLUMNS = Utils.getProperty("h2.consoleTableColumns", 500); + CONSOLE_MAX_PROCEDURES_LIST_COLUMNS = Utils.getProperty("h2.consoleProcedureColumns", 300); + CONSOLE_STREAM = Utils.getProperty("h2.consoleStream", true); + CONSOLE_TIMEOUT = Utils.getProperty("h2.consoleTimeout", 1800000); + DATASOURCE_TRACE_LEVEL = Utils.getProperty("h2.dataSourceTraceLevel", 1); + DELAY_WRONG_PASSWORD_MIN = Utils.getProperty("h2.delayWrongPasswordMin", 250); + DELAY_WRONG_PASSWORD_MAX = Utils.getProperty("h2.delayWrongPasswordMax", 4000); + JAVA_SYSTEM_COMPILER = Utils.getProperty("h2.javaSystemCompiler", true); + lobCloseBetweenReads = Utils.getProperty("h2.lobCloseBetweenReads", false); + LOB_FILES_PER_DIRECTORY = Utils.getProperty("h2.lobFilesPerDirectory", 256); + LOB_CLIENT_MAX_SIZE_MEMORY = Utils.getProperty("h2.lobClientMaxSizeMemory", 1048576); + MAX_FILE_RETRY = Math.max(1, Utils.getProperty("h2.maxFileRetry", 16)); + MAX_RECONNECT = Utils.getProperty("h2.maxReconnect", 3); + MAX_MEMORY_ROWS = getAutoScaledForMemoryProperty("h2.maxMemoryRows", 40000); + MAX_TRACE_DATA_LENGTH = (long)Utils.getProperty("h2.maxTraceDataLength", 65535); + MODIFY_ON_WRITE = Utils.getProperty("h2.modifyOnWrite", false); + NIO_LOAD_MAPPED = Utils.getProperty("h2.nioLoadMapped", false); + NIO_CLEANER_HACK = Utils.getProperty("h2.nioCleanerHack", false); + OBJECT_CACHE = Utils.getProperty("h2.objectCache", true); + OBJECT_CACHE_MAX_PER_ELEMENT_SIZE = Utils.getProperty("h2.objectCacheMaxPerElementSize", 4096); + + try { + OBJECT_CACHE_SIZE = MathUtils.nextPowerOf2(Utils.getProperty("h2.objectCacheSize", 1024)); + } catch (IllegalArgumentException var1) { + throw new IllegalStateException("Invalid h2.objectCacheSize", var1); + } + + OLD_RESULT_SET_GET_OBJECT = Utils.getProperty("h2.oldResultSetGetObject", !PREVIEW); + BIG_DECIMAL_IS_DECIMAL = Utils.getProperty("h2.bigDecimalIsDecimal", !PREVIEW); + RETURN_OFFSET_DATE_TIME = Utils.getProperty("h2.returnOffsetDateTime", PREVIEW); + PG_DEFAULT_CLIENT_ENCODING = Utils.getProperty("h2.pgClientEncoding", "UTF-8"); + PREFIX_TEMP_FILE = Utils.getProperty("h2.prefixTempFile", "h2.temp"); + FORCE_AUTOCOMMIT_OFF_ON_COMMIT = Utils.getProperty("h2.forceAutoCommitOffOnCommit", false); + SERVER_CACHED_OBJECTS = Utils.getProperty("h2.serverCachedObjects", 64); + SERVER_RESULT_SET_FETCH_SIZE = Utils.getProperty("h2.serverResultSetFetchSize", 100); + SOCKET_CONNECT_RETRY = Utils.getProperty("h2.socketConnectRetry", 16); + SOCKET_CONNECT_TIMEOUT = Utils.getProperty("h2.socketConnectTimeout", 2000); + SORT_BINARY_UNSIGNED = Utils.getProperty("h2.sortBinaryUnsigned", true); + SORT_UUID_UNSIGNED = Utils.getProperty("h2.sortUuidUnsigned", PREVIEW); + SORT_NULLS_HIGH = Utils.getProperty("h2.sortNullsHigh", false); + SPLIT_FILE_SIZE_SHIFT = (long)Utils.getProperty("h2.splitFileSizeShift", 30); + SYNC_METHOD = Utils.getProperty("h2.syncMethod", "sync"); + TRACE_IO = Utils.getProperty("h2.traceIO", false); + THREAD_DEADLOCK_DETECTOR = Utils.getProperty("h2.threadDeadlockDetector", false); + IMPLICIT_RELATIVE_PATH = Utils.getProperty("h2.implicitRelativePath", false); + URL_MAP = Utils.getProperty("h2.urlMap", (String)null); + USE_THREAD_CONTEXT_CLASS_LOADER = Utils.getProperty("h2.useThreadContextClassLoader", false); + serializeJavaObject = Utils.getProperty("h2.serializeJavaObject", true); + JAVA_OBJECT_SERIALIZER = Utils.getProperty("h2.javaObjectSerializer", (String)null); + CUSTOM_DATA_TYPES_HANDLER = Utils.getProperty("h2.customDataTypesHandler", (String)null); + AUTH_CONFIG_FILE = Utils.getProperty("h2.authConfigFile", (String)null); + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/WriteBuffer.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/WriteBuffer.java new file mode 100644 index 000000000..5471b5b72 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/WriteBuffer.java @@ -0,0 +1,333 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore; + +import org.h2.mvstore.DataUtils; + +import java.nio.ByteBuffer; + +/** + * An auto-resize buffer to write data into a ByteBuffer. + */ +public class WriteBuffer { + + /** + * The maximum size of the buffer in order to be re-used after a clear + * operation. + */ + private static final int MAX_REUSE_CAPACITY = 4 * 1024 * 1024; + + /** + * The minimum number of bytes to grow a buffer at a time. + */ + private static final int MIN_GROW = 1024 * 1024; + + /** + * The buffer that is used after a clear operation. + */ + private ByteBuffer reuse; + + /** + * The current buffer (may be replaced if it is too small). + */ + private ByteBuffer buff; + + public WriteBuffer(int initialSize) { + reuse = ByteBuffer.allocate(initialSize); + buff = reuse; + } + + public WriteBuffer() { + this(MIN_GROW); + } + + /** + * Write a variable size integer. + * + * @param x the value + * @return this + */ + public WriteBuffer putVarInt(int x) { + DataUtils.writeVarInt(ensureCapacity(5), x); + return this; + } + + /** + * Write a variable size long. + * + * @param x the value + * @return this + */ + public WriteBuffer putVarLong(long x) { + DataUtils.writeVarLong(ensureCapacity(10), x); + return this; + } + + /** + * Write the characters of a string in a format similar to UTF-8. + * + * @param s the string + * @param len the number of characters to write + * @return this + */ + public WriteBuffer putStringData(String s, int len) { + ByteBuffer b = ensureCapacity(3 * len); + DataUtils.writeStringData(b, s, len); + return this; + } + + /** + * Put a byte. + * + * @param x the value + * @return this + */ + public WriteBuffer put(byte x) { + ensureCapacity(1).put(x); + return this; + } + + /** + * Put a character. + * + * @param x the value + * @return this + */ + public WriteBuffer putChar(char x) { + ensureCapacity(2).putChar(x); + return this; + } + + /** + * Put a short. + * + * @param x the value + * @return this + */ + public WriteBuffer putShort(short x) { + ensureCapacity(2).putShort(x); + return this; + } + + /** + * Put an integer. + * + * @param x the value + * @return this + */ + public WriteBuffer putInt(int x) { + ensureCapacity(4).putInt(x); + return this; + } + + /** + * Put a long. + * + * @param x the value + * @return this + */ + public WriteBuffer putLong(long x) { + ensureCapacity(8).putLong(x); + return this; + } + + /** + * Put a float. + * + * @param x the value + * @return this + */ + public WriteBuffer putFloat(float x) { + ensureCapacity(4).putFloat(x); + return this; + } + + /** + * Put a double. + * + * @param x the value + * @return this + */ + public WriteBuffer putDouble(double x) { + ensureCapacity(8).putDouble(x); + return this; + } + + /** + * Put a byte array. + * + * @param bytes the value + * @return this + */ + public WriteBuffer put(byte[] bytes) { + ensureCapacity(bytes.length).put(bytes); + return this; + } + + /** + * Put a byte array. + * + * @param bytes the value + * @param offset the source offset + * @param length the number of bytes + * @return this + */ + public WriteBuffer put(byte[] bytes, int offset, int length) { + ensureCapacity(length).put(bytes, offset, length); + return this; + } + + /** + * Put the contents of a byte buffer. + * + * @param src the source buffer + * @return this + */ + public WriteBuffer put(ByteBuffer src) { + ensureCapacity(src.remaining()).put(src); + return this; + } + + /** + * Set the limit, possibly growing the buffer. + * + * @param newLimit the new limit + * @return this + */ + public WriteBuffer limit(int newLimit) { + ensureCapacity(newLimit - buff.position()).limit(newLimit); + return this; + } + + /** + * Get the capacity. + * + * @return the capacity + */ + public int capacity() { + return buff.capacity(); + } + + /** + * Set the position. + * + * @param newPosition the new position + * @return the new position + */ + public WriteBuffer position(int newPosition) { + buff.position(newPosition); + return this; + } + + /** + * Get the limit. + * + * @return the limit + */ + public int limit() { + return buff.limit(); + } + + /** + * Get the current position. + * + * @return the position + */ + public int position() { + return buff.position(); + } + + /** + * Copy the data into the destination array. + * + * @param dst the destination array + * @return this + */ + public WriteBuffer get(byte[] dst) { + buff.get(dst); + return this; + } + + /** + * Update an integer at the given index. + * + * @param index the index + * @param value the value + * @return this + */ + public WriteBuffer putInt(int index, int value) { + buff.putInt(index, value); + return this; + } + + /** + * Update a short at the given index. + * + * @param index the index + * @param value the value + * @return this + */ + public WriteBuffer putShort(int index, short value) { + buff.putShort(index, value); + return this; + } + + /** + * Clear the buffer after use. + * + * @return this + */ + public WriteBuffer clear() { + if (buff.limit() > MAX_REUSE_CAPACITY) { + buff = reuse; + } else if (buff != reuse) { + reuse = buff; + } + buff.clear(); + return this; + } + + /** + * Get the byte buffer. + * + * @return the byte buffer + */ + public ByteBuffer getBuffer() { + return buff; + } + + private ByteBuffer ensureCapacity(int len) { + if (buff.remaining() < len) { + grow(len); + } + return buff; + } + + private void grow(int additional) { + ByteBuffer temp = buff; + int needed = additional - temp.remaining(); + // grow at least MIN_GROW + long grow = Math.max(needed, MIN_GROW); + // grow at least 50% of the current size + grow = Math.max(temp.capacity() / 2, grow); + // the new capacity is at most Integer.MAX_VALUE + int newCapacity = (int) Math.min(Integer.MAX_VALUE, temp.capacity() + grow); + if (newCapacity < needed) { + throw new OutOfMemoryError("Capacity: " + newCapacity + " needed: " + needed); + } + try { + buff = ByteBuffer.allocate(newCapacity); + } catch (OutOfMemoryError e) { + throw new OutOfMemoryError("Capacity: " + newCapacity); + } + temp.flip(); + buff.put(temp); + if (newCapacity <= MAX_REUSE_CAPACITY) { + reuse = buff; + } + } + +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/cache/CacheLongKeyLIRS.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/cache/CacheLongKeyLIRS.java new file mode 100644 index 000000000..c7d2c6435 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/cache/CacheLongKeyLIRS.java @@ -0,0 +1,1215 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.cache; + + + +import org.dizitart.no2.mvstore.compat.v1.mvstore.DataUtils; + +import java.lang.ref.WeakReference; +import java.util.*; + +/** + * A scan resistant cache that uses keys of type long. It is meant to cache + * objects that are relatively costly to acquire, for example file content. + *

+ * This implementation is multi-threading safe and supports concurrent access. + * Null keys or null values are not allowed. The map fill factor is at most 75%. + *

+ * Each entry is assigned a distinct memory size, and the cache will try to use + * at most the specified amount of memory. The memory unit is not relevant, + * however it is suggested to use bytes as the unit. + *

+ * This class implements an approximation of the LIRS replacement algorithm + * invented by Xiaodong Zhang and Song Jiang as described in + * http://www.cse.ohio-state.edu/~zhang/lirs-sigmetrics-02.html with a few + * smaller changes: An additional queue for non-resident entries is used, to + * prevent unbound memory usage. The maximum size of this queue is at most the + * size of the rest of the stack. About 6.25% of the mapped entries are cold. + *

+ * Internally, the cache is split into a number of segments, and each segment is + * an individual LIRS cache. + *

+ * Accessed entries are only moved to the top of the stack if at least a number + * of other entries have been moved to the front (8 per segment by default). + * Write access and moving entries to the top of the stack is synchronized per + * segment. + * + * @author Thomas Mueller + * @param the value type + */ +public class CacheLongKeyLIRS { + + /** + * The maximum memory this cache should use. + */ + private long maxMemory; + + private final Segment[] segments; + + private final int segmentCount; + private final int segmentShift; + private final int segmentMask; + private final int stackMoveDistance; + private final int nonResidentQueueSize; + private final int nonResidentQueueSizeHigh; + + /** + * Create a new cache with the given memory size. + * + * @param config the configuration + */ + @SuppressWarnings("unchecked") + public CacheLongKeyLIRS(Config config) { + setMaxMemory(config.maxMemory); + this.nonResidentQueueSize = config.nonResidentQueueSize; + this.nonResidentQueueSizeHigh = config.nonResidentQueueSizeHigh; + DataUtils.checkArgument( + Integer.bitCount(config.segmentCount) == 1, + "The segment count must be a power of 2, is {0}", config.segmentCount); + this.segmentCount = config.segmentCount; + this.segmentMask = segmentCount - 1; + this.stackMoveDistance = config.stackMoveDistance; + segments = new Segment[segmentCount]; + clear(); + // use the high bits for the segment + this.segmentShift = 32 - Integer.bitCount(segmentMask); + } + + /** + * Remove all entries. + */ + public void clear() { + long max = getMaxItemSize(); + for (int i = 0; i < segmentCount; i++) { + segments[i] = new Segment<>(max, stackMoveDistance, 8, nonResidentQueueSize, + nonResidentQueueSizeHigh); + } + } + + /** + * Determines max size of the data item size to fit into cache + * @return data items size limit + */ + public long getMaxItemSize() { + return Math.max(1, maxMemory / segmentCount); + } + + private Entry find(long key) { + int hash = getHash(key); + return getSegment(hash).find(key, hash); + } + + /** + * Check whether there is a resident entry for the given key. This + * method does not adjust the internal state of the cache. + * + * @param key the key (may not be null) + * @return true if there is a resident entry + */ + public boolean containsKey(long key) { + Entry e = find(key); + return e != null && e.value != null; + } + + /** + * Get the value for the given key if the entry is cached. This method does + * not modify the internal state. + * + * @param key the key (may not be null) + * @return the value, or null if there is no resident entry + */ + public V peek(long key) { + Entry e = find(key); + return e == null ? null : e.getValue(); + } + + /** + * Add an entry to the cache using the average memory size. + * + * @param key the key (may not be null) + * @param value the value (may not be null) + * @return the old value, or null if there was no resident entry + */ + public V put(long key, V value) { + return put(key, value, sizeOf(value)); + } + + /** + * Add an entry to the cache. The entry may or may not exist in the + * cache yet. This method will usually mark unknown entries as cold and + * known entries as hot. + * + * @param key the key (may not be null) + * @param value the value (may not be null) + * @param memory the memory used for the given entry + * @return the old value, or null if there was no resident entry + */ + public V put(long key, V value, int memory) { + if (value == null) { + throw DataUtils.newIllegalArgumentException( + "The value may not be null"); + } + int hash = getHash(key); + int segmentIndex = getSegmentIndex(hash); + Segment s = segments[segmentIndex]; + // check whether resize is required: synchronize on s, to avoid + // concurrent resizes (concurrent reads read + // from the old segment) + synchronized (s) { + s = resizeIfNeeded(s, segmentIndex); + return s.put(key, hash, value, memory); + } + } + + private Segment resizeIfNeeded(Segment s, int segmentIndex) { + int newLen = s.getNewMapLen(); + if (newLen == 0) { + return s; + } + // another thread might have resized + // (as we retrieved the segment before synchronizing on it) + Segment s2 = segments[segmentIndex]; + if (s == s2) { + // no other thread resized, so we do + s = new Segment<>(s, newLen); + segments[segmentIndex] = s; + } + return s; + } + + /** + * Get the size of the given value. The default implementation returns 1. + * + * @param value the value + * @return the size + */ + @SuppressWarnings("unused") + protected int sizeOf(V value) { + return 1; + } + + /** + * Remove an entry. Both resident and non-resident entries can be + * removed. + * + * @param key the key (may not be null) + * @return the old value, or null if there was no resident entry + */ + public V remove(long key) { + int hash = getHash(key); + int segmentIndex = getSegmentIndex(hash); + Segment s = segments[segmentIndex]; + // check whether resize is required: synchronize on s, to avoid + // concurrent resizes (concurrent reads read + // from the old segment) + synchronized (s) { + s = resizeIfNeeded(s, segmentIndex); + return s.remove(key, hash); + } + } + + /** + * Get the memory used for the given key. + * + * @param key the key (may not be null) + * @return the memory, or 0 if there is no resident entry + */ + public int getMemory(long key) { + Entry e = find(key); + return e == null ? 0 : e.getMemory(); + } + + /** + * Get the value for the given key if the entry is cached. This method + * adjusts the internal state of the cache sometimes, to ensure commonly + * used entries stay in the cache. + * + * @param key the key (may not be null) + * @return the value, or null if there is no resident entry + */ + public V get(long key) { + int hash = getHash(key); + Segment s = getSegment(hash); + Entry e = s.find(key, hash); + return s.get(e); + } + + private Segment getSegment(int hash) { + return segments[getSegmentIndex(hash)]; + } + + private int getSegmentIndex(int hash) { + return (hash >>> segmentShift) & segmentMask; + } + + /** + * Get the hash code for the given key. The hash code is + * further enhanced to spread the values more evenly. + * + * @param key the key + * @return the hash code + */ + static int getHash(long key) { + int hash = (int) ((key >>> 32) ^ key); + // a supplemental secondary hash function + // to protect against hash codes that don't differ much + hash = ((hash >>> 16) ^ hash) * 0x45d9f3b; + hash = ((hash >>> 16) ^ hash) * 0x45d9f3b; + hash = (hash >>> 16) ^ hash; + return hash; + } + + /** + * Get the currently used memory. + * + * @return the used memory + */ + public long getUsedMemory() { + long x = 0; + for (Segment s : segments) { + x += s.usedMemory; + } + return x; + } + + /** + * Set the maximum memory this cache should use. This will not + * immediately cause entries to get removed however; it will only change + * the limit. To resize the internal array, call the clear method. + * + * @param maxMemory the maximum size (1 or larger) in bytes + */ + public void setMaxMemory(long maxMemory) { + DataUtils.checkArgument( + maxMemory > 0, + "Max memory must be larger than 0, is {0}", maxMemory); + this.maxMemory = maxMemory; + if (segments != null) { + long max = 1 + maxMemory / segments.length; + for (Segment s : segments) { + s.setMaxMemory(max); + } + } + } + + /** + * Get the maximum memory to use. + * + * @return the maximum memory + */ + public long getMaxMemory() { + return maxMemory; + } + + /** + * Get the entry set for all resident entries. + * + * @return the entry set + */ + public synchronized Set> entrySet() { + HashMap map = new HashMap<>(); + for (long k : keySet()) { + V value = peek(k); + if (value != null) { + map.put(k, value); + } + } + return map.entrySet(); + } + + /** + * Get the set of keys for resident entries. + * + * @return the set of keys + */ + public Set keySet() { + HashSet set = new HashSet<>(); + for (Segment s : segments) { + set.addAll(s.keySet()); + } + return set; + } + + /** + * Get the number of non-resident entries in the cache. + * + * @return the number of non-resident entries + */ + public int sizeNonResident() { + int x = 0; + for (Segment s : segments) { + x += s.queue2Size; + } + return x; + } + + /** + * Get the length of the internal map array. + * + * @return the size of the array + */ + public int sizeMapArray() { + int x = 0; + for (Segment s : segments) { + x += s.entries.length; + } + return x; + } + + /** + * Get the number of hot entries in the cache. + * + * @return the number of hot entries + */ + public int sizeHot() { + int x = 0; + for (Segment s : segments) { + x += s.mapSize - s.queueSize - s.queue2Size; + } + return x; + } + + /** + * Get the number of cache hits. + * + * @return the cache hits + */ + public long getHits() { + long x = 0; + for (Segment s : segments) { + x += s.hits; + } + return x; + } + + /** + * Get the number of cache misses. + * + * @return the cache misses + */ + public long getMisses() { + int x = 0; + for (Segment s : segments) { + x += s.misses; + } + return x; + } + + /** + * Get the number of resident entries. + * + * @return the number of entries + */ + public int size() { + int x = 0; + for (Segment s : segments) { + x += s.mapSize - s.queue2Size; + } + return x; + } + + /** + * Get the list of keys. This method allows to read the internal state of + * the cache. + * + * @param cold if true, only keys for the cold entries are returned + * @param nonResident true for non-resident entries + * @return the key list + */ + public List keys(boolean cold, boolean nonResident) { + ArrayList keys = new ArrayList<>(); + for (Segment s : segments) { + keys.addAll(s.keys(cold, nonResident)); + } + return keys; + } + + /** + * Get the values for all resident entries. + * + * @return the entry set + */ + public List values() { + ArrayList list = new ArrayList<>(); + for (long k : keySet()) { + V value = peek(k); + if (value != null) { + list.add(value); + } + } + return list; + } + + /** + * Check whether the cache is empty. + * + * @return true if it is empty + */ + public boolean isEmpty() { + return size() == 0; + } + + /** + * Check whether the given value is stored. + * + * @param value the value + * @return true if it is stored + */ + public boolean containsValue(V value) { + return getMap().containsValue(value); + } + + /** + * Convert this cache to a map. + * + * @return the map + */ + public Map getMap() { + HashMap map = new HashMap<>(); + for (long k : keySet()) { + V x = peek(k); + if (x != null) { + map.put(k, x); + } + } + return map; + } + + /** + * Add all elements of the map to this cache. + * + * @param m the map + */ + public void putAll(Map m) { + for (Map.Entry e : m.entrySet()) { + // copy only non-null entries + put(e.getKey(), e.getValue()); + } + } + + /** + * Loop through segments, trimming the non resident queue. + */ + public void trimNonResidentQueue() { + for (Segment s : segments) { + synchronized (s) { + s.trimNonResidentQueue(); + } + } + } + + /** + * A cache segment + * + * @param the value type + */ + private static class Segment { + + /** + * The number of (hot, cold, and non-resident) entries in the map. + */ + int mapSize; + + /** + * The size of the LIRS queue for resident cold entries. + */ + int queueSize; + + /** + * The size of the LIRS queue for non-resident cold entries. + */ + int queue2Size; + + /** + * The number of cache hits. + */ + long hits; + + /** + * The number of cache misses. + */ + long misses; + + /** + * The map array. The size is always a power of 2. + */ + final Entry[] entries; + + /** + * The currently used memory. + */ + long usedMemory; + + /** + * How many other item are to be moved to the top of the stack before + * the current item is moved. + */ + private final int stackMoveDistance; + + /** + * The maximum memory this cache should use in bytes. + */ + private long maxMemory; + + /** + * The bit mask that is applied to the key hash code to get the index in + * the map array. The mask is the length of the array minus one. + */ + private final int mask; + + /** + * Low watermark for the number of entries in the non-resident queue, + * as a factor of the number of entries in the map. + */ + private final int nonResidentQueueSize; + + /** + * High watermark for the number of entries in the non-resident queue, + * as a factor of the number of entries in the map. + */ + private final int nonResidentQueueSizeHigh; + + /** + * The stack of recently referenced elements. This includes all hot + * entries, and the recently referenced cold entries. Resident cold + * entries that were not recently referenced, as well as non-resident + * cold entries, are not in the stack. + *

+ * There is always at least one entry: the head entry. + */ + private final Entry stack; + + /** + * The number of entries in the stack. + */ + private int stackSize; + + /** + * The queue of resident cold entries. + *

+ * There is always at least one entry: the head entry. + */ + private final Entry queue; + + /** + * The queue of non-resident cold entries. + *

+ * There is always at least one entry: the head entry. + */ + private final Entry queue2; + + /** + * The number of times any item was moved to the top of the stack. + */ + private int stackMoveCounter; + + /** + * Create a new cache segment. + * @param maxMemory the maximum memory to use + * @param stackMoveDistance the number of other entries to be moved to + * the top of the stack before moving an entry to the top + * @param len the number of hash table buckets (must be a power of 2) + * @param nonResidentQueueSize the non-resident queue size low watermark factor + * @param nonResidentQueueSizeHigh the non-resident queue size high watermark factor + */ + Segment(long maxMemory, int stackMoveDistance, int len, + int nonResidentQueueSize, int nonResidentQueueSizeHigh) { + setMaxMemory(maxMemory); + this.stackMoveDistance = stackMoveDistance; + this.nonResidentQueueSize = nonResidentQueueSize; + this.nonResidentQueueSizeHigh = nonResidentQueueSizeHigh; + + // the bit mask has all bits set + mask = len - 1; + + // initialize the stack and queue heads + stack = new Entry<>(); + stack.stackPrev = stack.stackNext = stack; + queue = new Entry<>(); + queue.queuePrev = queue.queueNext = queue; + queue2 = new Entry<>(); + queue2.queuePrev = queue2.queueNext = queue2; + + @SuppressWarnings("unchecked") + Entry[] e = new Entry[len]; + entries = e; + } + + /** + * Create a new cache segment from an existing one. + * The caller must synchronize on the old segment, to avoid + * concurrent modifications. + * + * @param old the old segment + * @param len the number of hash table buckets (must be a power of 2) + */ + Segment(Segment old, int len) { + this(old.maxMemory, old.stackMoveDistance, len, + old.nonResidentQueueSize, old.nonResidentQueueSizeHigh); + hits = old.hits; + misses = old.misses; + Entry s = old.stack.stackPrev; + while (s != old.stack) { + Entry e = new Entry<>(s); + addToMap(e); + addToStack(e); + s = s.stackPrev; + } + s = old.queue.queuePrev; + while (s != old.queue) { + Entry e = find(s.key, getHash(s.key)); + if (e == null) { + e = new Entry<>(s); + addToMap(e); + } + addToQueue(queue, e); + s = s.queuePrev; + } + s = old.queue2.queuePrev; + while (s != old.queue2) { + Entry e = find(s.key, getHash(s.key)); + if (e == null) { + e = new Entry<>(s); + addToMap(e); + } + addToQueue(queue2, e); + s = s.queuePrev; + } + } + + /** + * Calculate the new number of hash table buckets if the internal map + * should be re-sized. + * + * @return 0 if no resizing is needed, or the new length + */ + int getNewMapLen() { + int len = mask + 1; + if (len * 3 < mapSize * 4 && len < (1 << 28)) { + // more than 75% usage + return len * 2; + } else if (len > 32 && len / 8 > mapSize) { + // less than 12% usage + return len / 2; + } + return 0; + } + + private void addToMap(Entry e) { + int index = getHash(e.key) & mask; + e.mapNext = entries[index]; + entries[index] = e; + usedMemory += e.getMemory(); + mapSize++; + } + + /** + * Get the value from the given entry. + * This method adjusts the internal state of the cache sometimes, + * to ensure commonly used entries stay in the cache. + * + * @param e the entry + * @return the value, or null if there is no resident entry + */ + synchronized V get(Entry e) { + V value = e == null ? null : e.getValue(); + if (value == null) { + // the entry was not found + // or it was a non-resident entry + misses++; + } else { + access(e); + hits++; + } + return value; + } + + /** + * Access an item, moving the entry to the top of the stack or front of + * the queue if found. + * + * @param e entry to record access for + */ + private void access(Entry e) { + if (e.isHot()) { + if (e != stack.stackNext && e.stackNext != null) { + if (stackMoveCounter - e.topMove > stackMoveDistance) { + // move a hot entry to the top of the stack + // unless it is already there + boolean wasEnd = e == stack.stackPrev; + removeFromStack(e); + if (wasEnd) { + // if moving the last entry, the last entry + // could now be cold, which is not allowed + pruneStack(); + } + addToStack(e); + } + } + } else { + V v = e.getValue(); + if (v != null) { + removeFromQueue(e); + if (e.reference != null) { + e.value = v; + e.reference = null; + usedMemory += e.memory; + } + if (e.stackNext != null) { + // resident, or even non-resident (weak value reference), + // cold entries become hot if they are on the stack + removeFromStack(e); + // which means a hot entry needs to become cold + // (this entry is cold, that means there is at least one + // more entry in the stack, which must be hot) + convertOldestHotToCold(); + } else { + // cold entries that are not on the stack + // move to the front of the queue + addToQueue(queue, e); + } + // in any case, the cold entry is moved to the top of the stack + addToStack(e); + // but if newly promoted cold/non-resident is the only entry on a stack now + // that means last one is cold, need to prune + pruneStack(); + } + } + } + + /** + * Add an entry to the cache. The entry may or may not exist in the + * cache yet. This method will usually mark unknown entries as cold and + * known entries as hot. + * + * @param key the key (may not be null) + * @param hash the hash + * @param value the value (may not be null) + * @param memory the memory used for the given entry + * @return the old value, or null if there was no resident entry + */ + synchronized V put(long key, int hash, V value, int memory) { + Entry e = find(key, hash); + boolean existed = e != null; + V old = null; + if (existed) { + old = e.getValue(); + remove(key, hash); + } + if (memory > maxMemory) { + // the new entry is too big to fit + return old; + } + e = new Entry<>(key, value, memory); + int index = hash & mask; + e.mapNext = entries[index]; + entries[index] = e; + usedMemory += memory; + if (usedMemory > maxMemory) { + // old entries needs to be removed + evict(); + // if the cache is full, the new entry is + // cold if possible + if (stackSize > 0) { + // the new cold entry is at the top of the queue + addToQueue(queue, e); + } + } + mapSize++; + // added entries are always added to the stack + addToStack(e); + if (existed) { + // if it was there before (even non-resident), it becomes hot + access(e); + } + return old; + } + + /** + * Remove an entry. Both resident and non-resident entries can be + * removed. + * + * @param key the key (may not be null) + * @param hash the hash + * @return the old value, or null if there was no resident entry + */ + synchronized V remove(long key, int hash) { + int index = hash & mask; + Entry e = entries[index]; + if (e == null) { + return null; + } + if (e.key == key) { + entries[index] = e.mapNext; + } else { + Entry last; + do { + last = e; + e = e.mapNext; + if (e == null) { + return null; + } + } while (e.key != key); + last.mapNext = e.mapNext; + } + V old = e.getValue(); + mapSize--; + usedMemory -= e.getMemory(); + if (e.stackNext != null) { + removeFromStack(e); + } + if (e.isHot()) { + // when removing a hot entry, the newest cold entry gets hot, + // so the number of hot entries does not change + e = queue.queueNext; + if (e != queue) { + removeFromQueue(e); + if (e.stackNext == null) { + addToStackBottom(e); + } + } + pruneStack(); + } else { + removeFromQueue(e); + } + return old; + } + + /** + * Evict cold entries (resident and non-resident) until the memory limit + * is reached. The new entry is added as a cold entry, except if it is + * the only entry. + */ + private void evict() { + do { + evictBlock(); + } while (usedMemory > maxMemory); + } + + private void evictBlock() { + // ensure there are not too many hot entries: right shift of 5 is + // division by 32, that means if there are only 1/32 (3.125%) or + // less cold entries, a hot entry needs to become cold + while (queueSize <= ((mapSize - queue2Size) >>> 5) && stackSize > 0) { + convertOldestHotToCold(); + } + // the oldest resident cold entries become non-resident + while (usedMemory > maxMemory && queueSize > 0) { + Entry e = queue.queuePrev; + usedMemory -= e.memory; + removeFromQueue(e); + e.reference = new WeakReference<>(e.value); + e.value = null; + addToQueue(queue2, e); + // the size of the non-resident-cold entries needs to be limited + trimNonResidentQueue(); + } + } + + void trimNonResidentQueue() { + int residentCount = mapSize - queue2Size; + int maxQueue2SizeHigh = nonResidentQueueSizeHigh * residentCount; + int maxQueue2Size = nonResidentQueueSize * residentCount; + while (queue2Size > maxQueue2Size) { + Entry e = queue2.queuePrev; + if (queue2Size <= maxQueue2SizeHigh) { + WeakReference reference = e.reference; + if (reference != null && reference.get() != null) { + break; // stop trimming if entry holds a value + } + } + int hash = getHash(e.key); + remove(e.key, hash); + } + } + + private void convertOldestHotToCold() { + // the last entry of the stack is known to be hot + Entry last = stack.stackPrev; + if (last == stack) { + // never remove the stack head itself (this would mean the + // internal structure of the cache is corrupt) + throw new IllegalStateException(); + } + // remove from stack - which is done anyway in the stack pruning, + // but we can do it here as well + removeFromStack(last); + // adding an entry to the queue will make it cold + addToQueue(queue, last); + pruneStack(); + } + + /** + * Ensure the last entry of the stack is cold. + */ + private void pruneStack() { + while (true) { + Entry last = stack.stackPrev; + // must stop at a hot entry or the stack head, + // but the stack head itself is also hot, so we + // don't have to test it + if (last.isHot()) { + break; + } + // the cold entry is still in the queue + removeFromStack(last); + } + } + + /** + * Try to find an entry in the map. + * + * @param key the key + * @param hash the hash + * @return the entry (might be a non-resident) + */ + Entry find(long key, int hash) { + int index = hash & mask; + Entry e = entries[index]; + while (e != null && e.key != key) { + e = e.mapNext; + } + return e; + } + + private void addToStack(Entry e) { + e.stackPrev = stack; + e.stackNext = stack.stackNext; + e.stackNext.stackPrev = e; + stack.stackNext = e; + stackSize++; + e.topMove = stackMoveCounter++; + } + + private void addToStackBottom(Entry e) { + e.stackNext = stack; + e.stackPrev = stack.stackPrev; + e.stackPrev.stackNext = e; + stack.stackPrev = e; + stackSize++; + } + + /** + * Remove the entry from the stack. The head itself must not be removed. + * + * @param e the entry + */ + private void removeFromStack(Entry e) { + e.stackPrev.stackNext = e.stackNext; + e.stackNext.stackPrev = e.stackPrev; + e.stackPrev = e.stackNext = null; + stackSize--; + } + + private void addToQueue(Entry q, Entry e) { + e.queuePrev = q; + e.queueNext = q.queueNext; + e.queueNext.queuePrev = e; + q.queueNext = e; + if (e.value != null) { + queueSize++; + } else { + queue2Size++; + } + } + + private void removeFromQueue(Entry e) { + e.queuePrev.queueNext = e.queueNext; + e.queueNext.queuePrev = e.queuePrev; + e.queuePrev = e.queueNext = null; + if (e.value != null) { + queueSize--; + } else { + queue2Size--; + } + } + + /** + * Get the list of keys. This method allows to read the internal state + * of the cache. + * + * @param cold if true, only keys for the cold entries are returned + * @param nonResident true for non-resident entries + * @return the key list + */ + synchronized List keys(boolean cold, boolean nonResident) { + ArrayList keys = new ArrayList<>(); + if (cold) { + Entry start = nonResident ? queue2 : queue; + for (Entry e = start.queueNext; e != start; + e = e.queueNext) { + keys.add(e.key); + } + } else { + for (Entry e = stack.stackNext; e != stack; + e = e.stackNext) { + keys.add(e.key); + } + } + return keys; + } + + /** + * Get the set of keys for resident entries. + * + * @return the set of keys + */ + synchronized Set keySet() { + HashSet set = new HashSet<>(); + for (Entry e = stack.stackNext; e != stack; e = e.stackNext) { + set.add(e.key); + } + for (Entry e = queue.queueNext; e != queue; e = e.queueNext) { + set.add(e.key); + } + return set; + } + + /** + * Set the maximum memory this cache should use. This will not + * immediately cause entries to get removed however; it will only change + * the limit. To resize the internal array, call the clear method. + * + * @param maxMemory the maximum size (1 or larger) in bytes + */ + void setMaxMemory(long maxMemory) { + this.maxMemory = maxMemory; + } + + } + + /** + * A cache entry. Each entry is either hot (low inter-reference recency; + * LIR), cold (high inter-reference recency; HIR), or non-resident-cold. Hot + * entries are in the stack only. Cold entries are in the queue, and may be + * in the stack. Non-resident-cold entries have their value set to null and + * are in the stack and in the non-resident queue. + * + * @param the value type + */ + static class Entry { + + /** + * The key. + */ + final long key; + + /** + * The value. Set to null for non-resident-cold entries. + */ + V value; + + /** + * Weak reference to the value. Set to null for resident entries. + */ + WeakReference reference; + + /** + * The estimated memory used. + */ + final int memory; + + /** + * When the item was last moved to the top of the stack. + */ + int topMove; + + /** + * The next entry in the stack. + */ + Entry stackNext; + + /** + * The previous entry in the stack. + */ + Entry stackPrev; + + /** + * The next entry in the queue (either the resident queue or the + * non-resident queue). + */ + Entry queueNext; + + /** + * The previous entry in the queue. + */ + Entry queuePrev; + + /** + * The next entry in the map (the chained entry). + */ + Entry mapNext; + + + Entry() { + this(0L, null, 0); + } + + Entry(long key, V value, int memory) { + this.key = key; + this.memory = memory; + this.value = value; + } + + Entry(Entry old) { + this(old.key, old.value, old.memory); + this.reference = old.reference; + this.topMove = old.topMove; + } + + /** + * Whether this entry is hot. Cold entries are in one of the two queues. + * + * @return whether the entry is hot + */ + boolean isHot() { + return queueNext == null; + } + + V getValue() { + return value == null ? reference.get() : value; + } + + int getMemory() { + return value == null ? 0 : memory; + } + } + + /** + * The cache configuration. + */ + public static class Config { + + /** + * The maximum memory to use (1 or larger). + */ + public long maxMemory = 1; + + /** + * The number of cache segments (must be a power of 2). + */ + public int segmentCount = 16; + + /** + * How many other item are to be moved to the top of the stack before + * the current item is moved. + */ + public int stackMoveDistance = 32; + + /** + * Low water mark for the number of entries in the non-resident queue, + * as a factor of the number of all other entries in the map. + */ + public final int nonResidentQueueSize = 3; + + /** + * High watermark for the number of entries in the non-resident queue, + * as a factor of the number of all other entries in the map + */ + public final int nonResidentQueueSizeHigh = 12; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/cache/FilePathCache.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/cache/FilePathCache.java new file mode 100644 index 000000000..61a5c86d1 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/cache/FilePathCache.java @@ -0,0 +1,182 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.cache; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FileBase; +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePath; +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePathWrapper; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; + +/** + * A file with a read cache. + */ +public class FilePathCache extends FilePathWrapper { + + /** + * The instance. + */ + public static final FilePathCache INSTANCE = new FilePathCache(); + + /** + * Register the file system. + */ + static { + FilePath.register(INSTANCE); + } + + public static FileChannel wrap(FileChannel f) { + return new FileCache(f); + } + + @Override + public FileChannel open(String mode) throws IOException { + return new FileCache(getBase().open(mode)); + } + + @Override + public String getScheme() { + return "cache"; + } + + /** + * A file with a read cache. + */ + public static class FileCache extends FileBase { + + private static final int CACHE_BLOCK_SIZE = 4 * 1024; + private final FileChannel base; + + private final CacheLongKeyLIRS cache; + + { + CacheLongKeyLIRS.Config cc = new CacheLongKeyLIRS.Config(); + // 1 MB cache size + cc.maxMemory = 1024 * 1024; + cache = new CacheLongKeyLIRS<>(cc); + } + + FileCache(FileChannel base) { + this.base = base; + } + + @Override + protected void implCloseChannel() throws IOException { + base.close(); + } + + @Override + public FileChannel position(long newPosition) throws IOException { + base.position(newPosition); + return this; + } + + @Override + public long position() throws IOException { + return base.position(); + } + + @Override + public int read(ByteBuffer dst) throws IOException { + return base.read(dst); + } + + @Override + public synchronized int read(ByteBuffer dst, long position) throws IOException { + long cachePos = getCachePos(position); + int off = (int) (position - cachePos); + int len = CACHE_BLOCK_SIZE - off; + len = Math.min(len, dst.remaining()); + ByteBuffer buff = cache.get(cachePos); + if (buff == null) { + buff = ByteBuffer.allocate(CACHE_BLOCK_SIZE); + long pos = cachePos; + while (true) { + int read = base.read(buff, pos); + if (read <= 0) { + break; + } + if (buff.remaining() == 0) { + break; + } + pos += read; + } + int read = buff.position(); + if (read == CACHE_BLOCK_SIZE) { + cache.put(cachePos, buff, CACHE_BLOCK_SIZE); + } else { + if (read <= 0) { + return -1; + } + len = Math.min(len, read - off); + } + } + dst.put(buff.array(), off, len); + return len == 0 ? -1 : len; + } + + private static long getCachePos(long pos) { + return (pos / CACHE_BLOCK_SIZE) * CACHE_BLOCK_SIZE; + } + + @Override + public long size() throws IOException { + return base.size(); + } + + @Override + public synchronized FileChannel truncate(long newSize) throws IOException { + cache.clear(); + base.truncate(newSize); + return this; + } + + @Override + public synchronized int write(ByteBuffer src, long position) throws IOException { + clearCache(src, position); + return base.write(src, position); + } + + @Override + public synchronized int write(ByteBuffer src) throws IOException { + clearCache(src, position()); + return base.write(src); + } + + private void clearCache(ByteBuffer src, long position) { + if (cache.size() > 0) { + int len = src.remaining(); + long p = getCachePos(position); + while (len > 0) { + cache.remove(p); + p += CACHE_BLOCK_SIZE; + len -= CACHE_BLOCK_SIZE; + } + } + } + + @Override + public void force(boolean metaData) throws IOException { + base.force(metaData); + } + + @Override + public FileLock tryLock(long position, long size, boolean shared) + throws IOException { + return base.tryLock(position, size, shared); + } + + @Override + public String toString() { + return "cache:" + base.toString(); + } + + } + +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/CompressDeflate.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/CompressDeflate.java new file mode 100644 index 000000000..250f33aa8 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/CompressDeflate.java @@ -0,0 +1,84 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.compress; + +import org.h2.message.DbException; + +import java.util.StringTokenizer; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +public class CompressDeflate implements Compressor { + private int level = -1; + private int strategy = 0; + + public CompressDeflate() { + } + + public void setOptions(String var1) { + if (var1 != null) { + try { + StringTokenizer var2 = new StringTokenizer(var1); + + while(var2.hasMoreElements()) { + String var3 = var2.nextToken(); + if (!"level".equals(var3) && !"l".equals(var3)) { + if ("strategy".equals(var3) || "s".equals(var3)) { + this.strategy = Integer.parseInt(var2.nextToken()); + } + } else { + this.level = Integer.parseInt(var2.nextToken()); + } + + Deflater var4 = new Deflater(this.level); + var4.setStrategy(this.strategy); + } + + } catch (Exception var5) { + throw DbException.get(90102, var1); + } + } + } + + public int compress(byte[] var1, int var2, byte[] var3, int var4) { + Deflater var5 = new Deflater(this.level); + var5.setStrategy(this.strategy); + var5.setInput(var1, 0, var2); + var5.finish(); + int var6 = var5.deflate(var3, var4, var3.length - var4); + if (var6 == 0) { + this.strategy = 0; + this.level = -1; + return this.compress(var1, var2, var3, var4); + } else { + var5.end(); + return var4 + var6; + } + } + + public int getAlgorithm() { + return 2; + } + + public void expand(byte[] var1, int var2, int var3, byte[] var4, int var5, int var6) { + Inflater var7 = new Inflater(); + var7.setInput(var1, var2, var3); + var7.finished(); + + try { + int var8 = var7.inflate(var4, var5, var6); + if (var8 != var6) { + throw new DataFormatException(var8 + " " + var6); + } + } catch (DataFormatException var9) { + throw DbException.get(90104, var9, new String[0]); + } + + var7.end(); + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/CompressLZF.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/CompressLZF.java new file mode 100644 index 000000000..62d0adae7 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/CompressLZF.java @@ -0,0 +1,272 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.compress; + +import java.nio.ByteBuffer; + +public final class CompressLZF implements Compressor { + private static final int HASH_SIZE = 16384; + private static final int MAX_LITERAL = 32; + private static final int MAX_OFF = 8192; + private static final int MAX_REF = 264; + private int[] cachedHashTable; + + public CompressLZF() { + } + + public void setOptions(String var1) { + } + + private static int first(byte[] var0, int var1) { + return var0[var1] << 8 | var0[var1 + 1] & 255; + } + + private static int first(ByteBuffer var0, int var1) { + return var0.get(var1) << 8 | var0.get(var1 + 1) & 255; + } + + private static int next(int var0, byte[] var1, int var2) { + return var0 << 8 | var1[var2 + 2] & 255; + } + + private static int next(int var0, ByteBuffer var1, int var2) { + return var0 << 8 | var1.get(var2 + 2) & 255; + } + + private static int hash(int var0) { + return var0 * 2777 >> 9 & 16383; + } + + public int compress(byte[] var1, int var2, byte[] var3, int var4) { + int var5 = 0; + if (this.cachedHashTable == null) { + this.cachedHashTable = new int[16384]; + } + + int[] var6 = this.cachedHashTable; + int var7 = 0; + ++var4; + int var8 = first((byte[])var1, 0); + + while(true) { + while(var5 < var2 - 4) { + byte var9 = var1[var5 + 2]; + var8 = (var8 << 8) + (var9 & 255); + int var10 = hash(var8); + int var11 = var6[var10]; + var6[var10] = var5; + if (var11 < var5 && var11 > 0 && (var10 = var5 - var11 - 1) < 8192 && var1[var11 + 2] == var9 && var1[var11 + 1] == (byte)(var8 >> 8) && var1[var11] == (byte)(var8 >> 16)) { + int var12 = var2 - var5 - 2; + if (var12 > 264) { + var12 = 264; + } + + if (var7 == 0) { + --var4; + } else { + var3[var4 - var7 - 1] = (byte)(var7 - 1); + var7 = 0; + } + + int var13; + for(var13 = 3; var13 < var12 && var1[var11 + var13] == var1[var5 + var13]; ++var13) { + } + + var13 -= 2; + if (var13 < 7) { + var3[var4++] = (byte)((var10 >> 8) + (var13 << 5)); + } else { + var3[var4++] = (byte)((var10 >> 8) + 224); + var3[var4++] = (byte)(var13 - 7); + } + + var3[var4++] = (byte)var10; + ++var4; + var5 += var13; + var8 = first(var1, var5); + var8 = next(var8, var1, var5); + var6[hash(var8)] = var5++; + var8 = next(var8, var1, var5); + var6[hash(var8)] = var5++; + } else { + var3[var4++] = var1[var5++]; + ++var7; + if (var7 == 32) { + var3[var4 - var7 - 1] = (byte)(var7 - 1); + var7 = 0; + ++var4; + } + } + } + + while(var5 < var2) { + var3[var4++] = var1[var5++]; + ++var7; + if (var7 == 32) { + var3[var4 - var7 - 1] = (byte)(var7 - 1); + var7 = 0; + ++var4; + } + } + + var3[var4 - var7 - 1] = (byte)(var7 - 1); + if (var7 == 0) { + --var4; + } + + return var4; + } + } + + public int compress(ByteBuffer var1, int var2, byte[] var3, int var4) { + int var5 = var1.capacity() - var2; + if (this.cachedHashTable == null) { + this.cachedHashTable = new int[16384]; + } + + int[] var6 = this.cachedHashTable; + int var7 = 0; + ++var4; + int var8 = first((ByteBuffer)var1, 0); + + while(true) { + while(var2 < var5 - 4) { + byte var9 = var1.get(var2 + 2); + var8 = (var8 << 8) + (var9 & 255); + int var10 = hash(var8); + int var11 = var6[var10]; + var6[var10] = var2; + if (var11 < var2 && var11 > 0 && (var10 = var2 - var11 - 1) < 8192 && var1.get(var11 + 2) == var9 && var1.get(var11 + 1) == (byte)(var8 >> 8) && var1.get(var11) == (byte)(var8 >> 16)) { + int var12 = var5 - var2 - 2; + if (var12 > 264) { + var12 = 264; + } + + if (var7 == 0) { + --var4; + } else { + var3[var4 - var7 - 1] = (byte)(var7 - 1); + var7 = 0; + } + + int var13; + for(var13 = 3; var13 < var12 && var1.get(var11 + var13) == var1.get(var2 + var13); ++var13) { + } + + var13 -= 2; + if (var13 < 7) { + var3[var4++] = (byte)((var10 >> 8) + (var13 << 5)); + } else { + var3[var4++] = (byte)((var10 >> 8) + 224); + var3[var4++] = (byte)(var13 - 7); + } + + var3[var4++] = (byte)var10; + ++var4; + var2 += var13; + var8 = first(var1, var2); + var8 = next(var8, var1, var2); + var6[hash(var8)] = var2++; + var8 = next(var8, var1, var2); + var6[hash(var8)] = var2++; + } else { + var3[var4++] = var1.get(var2++); + ++var7; + if (var7 == 32) { + var3[var4 - var7 - 1] = (byte)(var7 - 1); + var7 = 0; + ++var4; + } + } + } + + while(var2 < var5) { + var3[var4++] = var1.get(var2++); + ++var7; + if (var7 == 32) { + var3[var4 - var7 - 1] = (byte)(var7 - 1); + var7 = 0; + ++var4; + } + } + + var3[var4 - var7 - 1] = (byte)(var7 - 1); + if (var7 == 0) { + --var4; + } + + return var4; + } + } + + public void expand(byte[] var1, int var2, int var3, byte[] var4, int var5, int var6) { + if (var2 >= 0 && var5 >= 0 && var6 >= 0) { + do { + int var7 = var1[var2++] & 255; + if (var7 < 32) { + ++var7; + System.arraycopy(var1, var2, var4, var5, var7); + var5 += var7; + var2 += var7; + } else { + int var8 = var7 >> 5; + if (var8 == 7) { + var8 += var1[var2++] & 255; + } + + var8 += 2; + var7 = -((var7 & 31) << 8) - 1; + var7 -= var1[var2++] & 255; + var7 += var5; + if (var5 + var8 >= var4.length) { + throw new ArrayIndexOutOfBoundsException(); + } + + for(int var9 = 0; var9 < var8; ++var9) { + var4[var5++] = var4[var7++]; + } + } + } while(var5 < var6); + + } else { + throw new IllegalArgumentException(); + } + } + + public static void expand(ByteBuffer var0, ByteBuffer var1) { + do { + int var2 = var0.get() & 255; + int var3; + if (var2 < 32) { + ++var2; + + for(var3 = 0; var3 < var2; ++var3) { + var1.put(var0.get()); + } + } else { + var3 = var2 >> 5; + if (var3 == 7) { + var3 += var0.get() & 255; + } + + var3 += 2; + var2 = -((var2 & 31) << 8) - 1; + var2 -= var0.get() & 255; + var2 += var1.position(); + + for(int var4 = 0; var4 < var3; ++var4) { + var1.put(var1.get(var2++)); + } + } + } while(var1.position() < var1.capacity()); + + } + + public int getAlgorithm() { + return 1; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/Compressor.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/Compressor.java new file mode 100644 index 000000000..1a94089ff --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/compress/Compressor.java @@ -0,0 +1,21 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.compress; + +public interface Compressor { + int NO = 0; + int LZF = 1; + int DEFLATE = 2; + + int getAlgorithm(); + + int compress(byte[] var1, int var2, byte[] var3, int var4); + + void expand(byte[] var1, int var2, int var3, byte[] var4, int var5, int var6); + + void setOptions(String var1); +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileBase.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileBase.java new file mode 100644 index 000000000..73baeea53 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileBase.java @@ -0,0 +1,82 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +public abstract class FileBase extends FileChannel { + public FileBase() { + } + + public abstract long size() throws IOException; + + public abstract long position() throws IOException; + + public abstract FileChannel position(long var1) throws IOException; + + public abstract int read(ByteBuffer var1) throws IOException; + + public abstract int write(ByteBuffer var1) throws IOException; + + public synchronized int read(ByteBuffer var1, long var2) throws IOException { + long var4 = this.position(); + this.position(var2); + int var6 = this.read(var1); + this.position(var4); + return var6; + } + + public synchronized int write(ByteBuffer var1, long var2) throws IOException { + long var4 = this.position(); + this.position(var2); + int var6 = this.write(var1); + this.position(var4); + return var6; + } + + public abstract FileChannel truncate(long var1) throws IOException; + + public void force(boolean var1) throws IOException { + } + + protected void implCloseChannel() throws IOException { + } + + public FileLock lock(long var1, long var3, boolean var5) throws IOException { + throw new UnsupportedOperationException(); + } + + public MappedByteBuffer map(FileChannel.MapMode var1, long var2, long var4) throws IOException { + throw new UnsupportedOperationException(); + } + + public long read(ByteBuffer[] var1, int var2, int var3) throws IOException { + throw new UnsupportedOperationException(); + } + + public long transferFrom(ReadableByteChannel var1, long var2, long var4) throws IOException { + throw new UnsupportedOperationException(); + } + + public long transferTo(long var1, long var3, WritableByteChannel var5) throws IOException { + throw new UnsupportedOperationException(); + } + + public FileLock tryLock(long var1, long var3, boolean var5) throws IOException { + throw new UnsupportedOperationException(); + } + + public long write(ByteBuffer[] var1, int var2, int var3) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileChannelInputStream.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileChannelInputStream.java new file mode 100644 index 000000000..c023a8b82 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileChannelInputStream.java @@ -0,0 +1,56 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +public class FileChannelInputStream extends InputStream { + private final FileChannel channel; + private final boolean closeChannel; + private ByteBuffer buffer; + private long pos; + + public FileChannelInputStream(FileChannel var1, boolean var2) { + this.channel = var1; + this.closeChannel = var2; + } + + public int read() throws IOException { + if (this.buffer == null) { + this.buffer = ByteBuffer.allocate(1); + } + + this.buffer.rewind(); + int var1 = this.channel.read(this.buffer, (long)(this.pos++)); + return var1 < 0 ? -1 : this.buffer.get(0) & 255; + } + + public int read(byte[] var1) throws IOException { + return this.read(var1, 0, var1.length); + } + + public int read(byte[] var1, int var2, int var3) throws IOException { + ByteBuffer var4 = ByteBuffer.wrap(var1, var2, var3); + int var5 = this.channel.read(var4, this.pos); + if (var5 == -1) { + return -1; + } else { + this.pos += (long)var5; + return var5; + } + } + + public void close() throws IOException { + if (this.closeChannel) { + this.channel.close(); + } + + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileChannelOutputStream.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileChannelOutputStream.java new file mode 100644 index 000000000..d481ea99d --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileChannelOutputStream.java @@ -0,0 +1,44 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +public class FileChannelOutputStream extends OutputStream { + private final FileChannel channel; + private final byte[] buffer = new byte[]{0}; + + public FileChannelOutputStream(FileChannel var1, boolean var2) throws IOException { + this.channel = var1; + if (var2) { + var1.position(var1.size()); + } else { + var1.position(0L); + var1.truncate(0L); + } + + } + + public void write(int var1) throws IOException { + this.buffer[0] = (byte)var1; + FileUtils.writeFully(this.channel, ByteBuffer.wrap(this.buffer)); + } + + public void write(byte[] var1) throws IOException { + FileUtils.writeFully(this.channel, ByteBuffer.wrap(var1)); + } + + public void write(byte[] var1, int var2, int var3) throws IOException { + FileUtils.writeFully(this.channel, ByteBuffer.wrap(var1, var2, var3)); + } + + public void close() throws IOException { + this.channel.close(); + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileDisk.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileDisk.java new file mode 100644 index 000000000..38f96c9db --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileDisk.java @@ -0,0 +1,96 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.SysProperties; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.NonWritableChannelException; + + +class FileDisk extends FileBase { + private final RandomAccessFile file; + private final String name; + private final boolean readOnly; + + FileDisk(String var1, String var2) throws FileNotFoundException { + this.file = new RandomAccessFile(var1, var2); + this.name = var1; + this.readOnly = var2.equals("r"); + } + + public void force(boolean var1) throws IOException { + String var2 = SysProperties.SYNC_METHOD; + if (!"".equals(var2)) { + if ("sync".equals(var2)) { + this.file.getFD().sync(); + } else if ("force".equals(var2)) { + this.file.getChannel().force(true); + } else if ("forceFalse".equals(var2)) { + this.file.getChannel().force(false); + } else { + this.file.getFD().sync(); + } + } + + } + + public FileChannel truncate(long var1) throws IOException { + if (this.readOnly) { + throw new NonWritableChannelException(); + } else { + this.file.getChannel().truncate(var1); + return this; + } + } + + public synchronized FileLock tryLock(long var1, long var3, boolean var5) throws IOException { + return this.file.getChannel().tryLock(var1, var3, var5); + } + + public void implCloseChannel() throws IOException { + this.file.close(); + } + + public long position() throws IOException { + return this.file.getFilePointer(); + } + + public long size() throws IOException { + return this.file.length(); + } + + public int read(ByteBuffer var1) throws IOException { + int var2 = this.file.read(var1.array(), var1.arrayOffset() + var1.position(), var1.remaining()); + if (var2 > 0) { + var1.position(var1.position() + var2); + } + + return var2; + } + + public FileChannel position(long var1) throws IOException { + this.file.seek(var1); + return this; + } + + public int write(ByteBuffer var1) throws IOException { + int var2 = var1.remaining(); + this.file.write(var1.array(), var1.arrayOffset() + var1.position(), var2); + var1.position(var1.position() + var2); + return var2; + } + + public String toString() { + return this.name; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileNio.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileNio.java new file mode 100644 index 000000000..5e7fd4424 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileNio.java @@ -0,0 +1,90 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.NonWritableChannelException; + +class FileNio extends FileBase { + private final String name; + private final FileChannel channel; + + FileNio(String var1, String var2) throws IOException { + this.name = var1; + this.channel = (new RandomAccessFile(var1, var2)).getChannel(); + } + + public void implCloseChannel() throws IOException { + this.channel.close(); + } + + public long position() throws IOException { + return this.channel.position(); + } + + public long size() throws IOException { + return this.channel.size(); + } + + public int read(ByteBuffer var1) throws IOException { + return this.channel.read(var1); + } + + public FileChannel position(long var1) throws IOException { + this.channel.position(var1); + return this; + } + + public int read(ByteBuffer var1, long var2) throws IOException { + return this.channel.read(var1, var2); + } + + public int write(ByteBuffer var1, long var2) throws IOException { + return this.channel.write(var1, var2); + } + + public FileChannel truncate(long var1) throws IOException { + long var3 = this.channel.size(); + if (var1 < var3) { + long var5 = this.channel.position(); + this.channel.truncate(var1); + long var7 = this.channel.position(); + if (var5 < var1) { + if (var7 != var5) { + this.channel.position(var5); + } + } else if (var7 > var1) { + this.channel.position(var1); + } + } + + return this; + } + + public void force(boolean var1) throws IOException { + this.channel.force(var1); + } + + public int write(ByteBuffer var1) throws IOException { + try { + return this.channel.write(var1); + } catch (NonWritableChannelException var3) { + throw new IOException("read only"); + } + } + + public synchronized FileLock tryLock(long var1, long var3, boolean var5) throws IOException { + return this.channel.tryLock(var1, var3, var5); + } + + public String toString() { + return "nio:" + this.name; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePath.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePath.java new file mode 100644 index 000000000..a813157b9 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePath.java @@ -0,0 +1,161 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import org.h2.util.MathUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public abstract class FilePath { + private static FilePath defaultProvider; + private static ConcurrentHashMap providers; + private static String tempRandom; + private static long tempSequence; + protected String name; + + public FilePath() { + } + + public static FilePath get(String var0) { + var0 = var0.replace('\\', '/'); + int var1 = var0.indexOf(58); + registerDefaultProviders(); + if (var1 < 2) { + return defaultProvider.getPath(var0); + } else { + String var2 = var0.substring(0, var1); + FilePath var3 = providers.get(var2); + if (var3 == null) { + var3 = defaultProvider; + } + + return var3.getPath(var0); + } + } + + private static void registerDefaultProviders() { + if (providers == null || defaultProvider == null) { + ConcurrentHashMap var0 = new ConcurrentHashMap(); + String[] var1 = new String[] { + "org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePathDisk", + "org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePathNio", + "org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FilePathEncrypt", + "org.h2.store.fs.FilePathMem", + "org.h2.store.fs.FilePathMemLZF", + "org.h2.store.fs.FilePathNioMem", + "org.h2.store.fs.FilePathNioMemLZF", + "org.h2.store.fs.FilePathSplit", + "org.h2.store.fs.FilePathNioMapped", + "org.h2.store.fs.FilePathAsync", + "org.h2.store.fs.FilePathZip", + "org.h2.store.fs.FilePathRetryOnInterrupt" + }; + int var2 = var1.length; + + for(int var3 = 0; var3 < var2; ++var3) { + String var4 = var1[var3]; + + try { + FilePath var5 = (FilePath)Class.forName(var4).getDeclaredConstructor().newInstance(); + var0.put(var5.getScheme(), var5); + if (defaultProvider == null) { + defaultProvider = var5; + } + } catch (Exception var6) { + } + } + + providers = var0; + } + + } + + public static void register(FilePath var0) { + registerDefaultProviders(); + providers.put(var0.getScheme(), var0); + } + + public static void unregister(FilePath var0) { + registerDefaultProviders(); + providers.remove(var0.getScheme()); + } + + public abstract long size(); + + public abstract void moveTo(FilePath var1, boolean var2); + + public abstract boolean createFile(); + + public abstract boolean exists(); + + public abstract void delete(); + + public abstract List newDirectoryStream(); + + public abstract FilePath toRealPath(); + + public abstract FilePath getParent(); + + public abstract boolean isDirectory(); + + public abstract boolean isAbsolute(); + + public abstract long lastModified(); + + public abstract boolean canWrite(); + + public abstract void createDirectory(); + + public String getName() { + int var1 = Math.max(this.name.indexOf(58), this.name.lastIndexOf(47)); + return var1 < 0 ? this.name : this.name.substring(var1 + 1); + } + + public abstract OutputStream newOutputStream(boolean var1) throws IOException; + + public abstract FileChannel open(String var1) throws IOException; + + public abstract InputStream newInputStream() throws IOException; + + public abstract boolean setReadOnly(); + + public FilePath createTempFile(String var1, boolean var2) throws IOException { + while(true) { + FilePath var3 = this.getPath(this.name + getNextTempFileNamePart(false) + var1); + if (!var3.exists() && var3.createFile()) { + var3.open("rw").close(); + return var3; + } + + getNextTempFileNamePart(true); + } + } + + protected static synchronized String getNextTempFileNamePart(boolean var0) { + if (var0 || tempRandom == null) { + tempRandom = MathUtils.randomInt(Integer.MAX_VALUE) + "."; + } + + return tempRandom + tempSequence++; + } + + public String toString() { + return this.name; + } + + public abstract String getScheme(); + + public abstract FilePath getPath(String var1); + + public FilePath unwrap() { + return this; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathDisk.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathDisk.java new file mode 100644 index 000000000..3e4341827 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathDisk.java @@ -0,0 +1,368 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.SysProperties; +import org.dizitart.no2.mvstore.compat.v1.mvstore.util.IOUtils; +import org.h2.message.DbException; + +import java.io.*; +import java.net.URL; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + +public class FilePathDisk extends FilePath { + private static final String CLASSPATH_PREFIX = "classpath:"; + + public FilePathDisk() { + } + + public FilePathDisk getPath(String var1) { + FilePathDisk var2 = new FilePathDisk(); + var2.name = translateFileName(var1); + return var2; + } + + public long size() { + if (this.name.startsWith("classpath:")) { + try { + String var1 = this.name.substring("classpath:".length()); + if (!var1.startsWith("/")) { + var1 = "/" + var1; + } + + URL var2 = this.getClass().getResource(var1); + return var2 != null ? new File(var2.getPath()).length() : 0L; + } catch (Exception var3) { + return 0L; + } + } else { + return (new File(this.name)).length(); + } + } + + protected static String translateFileName(String var0) { + var0 = var0.replace('\\', '/'); + if (var0.startsWith("file:")) { + var0 = var0.substring("file:".length()); + } + + return expandUserHomeDirectory(var0); + } + + public static String expandUserHomeDirectory(String var0) { + if (var0.startsWith("~") && (var0.length() == 1 || var0.startsWith("~/"))) { + String var1 = SysProperties.USER_HOME; + var0 = var1 + var0.substring(1); + } + + return var0; + } + + public void moveTo(FilePath var1, boolean var2) { + File var3 = new File(this.name); + File var4 = new File(var1.name); + if (!var3.getAbsolutePath().equals(var4.getAbsolutePath())) { + if (!var3.exists()) { + throw DbException.get(90024, this.name + " (not found)", var1.name); + } else if (var2) { + boolean var7 = var3.renameTo(var4); + if (!var7) { + throw DbException.get(90024, this.name, var1.name); + } + } else if (var4.exists()) { + throw DbException.get(90024, this.name, var1 + " (exists)"); + } else { + for(int var5 = 0; var5 < SysProperties.MAX_FILE_RETRY; ++var5) { + IOUtils.trace("rename", this.name + " >" + var1, (Object)null); + boolean var6 = var3.renameTo(var4); + if (var6) { + return; + } + + wait(var5); + } + + throw DbException.get(90024, this.name, var1.name); + } + } + } + + private static void wait(int var0) { + if (var0 == 8) { + System.gc(); + } + + try { + long var1 = (long)Math.min(256, var0 * var0); + Thread.sleep(var1); + } catch (InterruptedException var3) { + } + + } + + public boolean createFile() { + File var1 = new File(this.name); + int var2 = 0; + + while(var2 < SysProperties.MAX_FILE_RETRY) { + try { + return var1.createNewFile(); + } catch (IOException var4) { + wait(var2); + ++var2; + } + } + + return false; + } + + public boolean exists() { + return (new File(this.name)).exists(); + } + + public void delete() { + File var1 = new File(this.name); + + for(int var2 = 0; var2 < SysProperties.MAX_FILE_RETRY; ++var2) { + IOUtils.trace("delete", this.name, (Object)null); + boolean var3 = var1.delete(); + if (var3 || !var1.exists()) { + return; + } + + wait(var2); + } + + throw DbException.get(90025, this.name); + } + + public List newDirectoryStream() { + ArrayList var1 = new ArrayList(); + File var2 = new File(this.name); + + try { + String[] var3 = var2.list(); + if (var3 != null) { + String var4 = var2.getCanonicalPath(); + if (!var4.endsWith(SysProperties.FILE_SEPARATOR)) { + var4 = var4 + SysProperties.FILE_SEPARATOR; + } + + var1.ensureCapacity(var3.length); + String[] var5 = var3; + int var6 = var3.length; + + for(int var7 = 0; var7 < var6; ++var7) { + String var8 = var5[var7]; + var1.add(this.getPath(var4 + var8)); + } + } + + return var1; + } catch (IOException var9) { + throw DbException.convertIOException(var9, this.name); + } + } + + public boolean canWrite() { + return canWriteInternal(new File(this.name)); + } + + public boolean setReadOnly() { + File var1 = new File(this.name); + return var1.setReadOnly(); + } + + public FilePathDisk toRealPath() { + try { + String var1 = (new File(this.name)).getCanonicalPath(); + return this.getPath(var1); + } catch (IOException var2) { + throw DbException.convertIOException(var2, this.name); + } + } + + public FilePath getParent() { + String var1 = (new File(this.name)).getParent(); + return var1 == null ? null : this.getPath(var1); + } + + public boolean isDirectory() { + return (new File(this.name)).isDirectory(); + } + + public boolean isAbsolute() { + return (new File(this.name)).isAbsolute(); + } + + public long lastModified() { + return (new File(this.name)).lastModified(); + } + + private static boolean canWriteInternal(File var0) { + try { + if (!var0.canWrite()) { + return false; + } + } catch (Exception var16) { + return false; + } + + RandomAccessFile var1 = null; + + boolean var3; + try { + var1 = new RandomAccessFile(var0, "rw"); + boolean var2 = true; + return var2; + } catch (FileNotFoundException var14) { + var3 = false; + } finally { + if (var1 != null) { + try { + var1.close(); + } catch (IOException var13) { + } + } + + } + + return var3; + } + + public void createDirectory() { + File var1 = new File(this.name); + + for(int var2 = 0; var2 < SysProperties.MAX_FILE_RETRY; ++var2) { + if (var1.exists()) { + if (var1.isDirectory()) { + return; + } + + throw DbException.get(90062, this.name + " (a file with this name already exists)"); + } + + if (var1.mkdir()) { + return; + } + + wait(var2); + } + + throw DbException.get(90062, this.name); + } + + public OutputStream newOutputStream(boolean var1) throws IOException { + try { + File var2 = new File(this.name); + File var3 = var2.getParentFile(); + if (var3 != null) { + FileUtils.createDirectories(var3.getAbsolutePath()); + } + + FileOutputStream var4 = new FileOutputStream(this.name, var1); + IOUtils.trace("openFileOutputStream", this.name, var4); + return var4; + } catch (IOException var5) { + freeMemoryAndFinalize(); + return new FileOutputStream(this.name); + } + } + + public InputStream newInputStream() throws IOException { + if (this.name.matches("[a-zA-Z]{2,19}:.*")) { + if (this.name.startsWith("classpath:")) { + String var4 = this.name.substring("classpath:".length()); + if (!var4.startsWith("/")) { + var4 = "/" + var4; + } + + InputStream var2 = this.getClass().getResourceAsStream(var4); + if (var2 == null) { + var2 = Thread.currentThread().getContextClassLoader().getResourceAsStream(var4.substring(1)); + } + + if (var2 == null) { + throw new FileNotFoundException("resource " + var4); + } else { + return var2; + } + } else { + URL var3 = new URL(this.name); + return var3.openStream(); + } + } else { + FileInputStream var1 = new FileInputStream(this.name); + IOUtils.trace("openFileInputStream", this.name, var1); + return var1; + } + } + + static void freeMemoryAndFinalize() { + IOUtils.trace("freeMemoryAndFinalize", (String)null, (Object)null); + Runtime var0 = Runtime.getRuntime(); + long var1 = var0.freeMemory(); + + for(int var3 = 0; var3 < 16; ++var3) { + var0.gc(); + long var4 = var0.freeMemory(); + var0.runFinalization(); + if (var4 == var1) { + break; + } + + var1 = var4; + } + + } + + public FileChannel open(String var1) throws IOException { + FileDisk var2; + try { + var2 = new FileDisk(this.name, var1); + IOUtils.trace("open", this.name, var2); + } catch (IOException var6) { + freeMemoryAndFinalize(); + + try { + var2 = new FileDisk(this.name, var1); + } catch (IOException var5) { + throw var6; + } + } + + return var2; + } + + public String getScheme() { + return "file"; + } + + public FilePath createTempFile(String var1, boolean var2) throws IOException { + String var3 = this.name + "."; + String var4 = (new File(var3)).getName(); + File var5; + if (var2) { + var5 = new File(System.getProperty("java.io.tmpdir", ".")); + } else { + var5 = (new File(var3)).getAbsoluteFile().getParentFile(); + } + + FileUtils.createDirectories(var5.getAbsolutePath()); + + while(true) { + File var6 = new File(var5, var4 + getNextTempFileNamePart(false) + var1); + if (!var6.exists() && var6.createNewFile()) { + return get(var6.getCanonicalPath()); + } + + getNextTempFileNamePart(true); + } + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathEncrypt.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathEncrypt.java new file mode 100644 index 000000000..718c6d700 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathEncrypt.java @@ -0,0 +1,441 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import org.h2.security.AES; +import org.h2.security.BlockCipher; +import org.h2.security.SHA256; +import org.h2.util.MathUtils; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +public class FilePathEncrypt extends FilePathWrapper { + private static final String SCHEME = "encrypt"; + + public FilePathEncrypt() { + } + + public static void register() { + FilePath.register(new FilePathEncrypt()); + } + + public FileChannel open(String var1) throws IOException { + String[] var2 = this.parse(this.name); + FileChannel var3 = FileUtils.open(var2[1], var1); + byte[] var4 = var2[0].getBytes(StandardCharsets.UTF_8); + return new FileEncrypt(this.name, var4, var3); + } + + public String getScheme() { + return "encrypt"; + } + + protected String getPrefix() { + String[] var1 = this.parse(this.name); + return this.getScheme() + ":" + var1[0] + ":"; + } + + public FilePath unwrap(String var1) { + return FilePath.get(this.parse(var1)[1]); + } + + public long size() { + long var1 = this.getBase().size() - 4096L; + var1 = Math.max(0L, var1); + if ((var1 & 4095L) != 0L) { + var1 -= 4096L; + } + + return var1; + } + + public OutputStream newOutputStream(boolean var1) throws IOException { + return new FileChannelOutputStream(this.open("rw"), var1); + } + + public InputStream newInputStream() throws IOException { + return new FileChannelInputStream(this.open("r"), true); + } + + private String[] parse(String var1) { + if (!var1.startsWith(this.getScheme())) { + throw new IllegalArgumentException(var1 + " doesn't start with " + this.getScheme()); + } else { + var1 = var1.substring(this.getScheme().length() + 1); + int var2 = var1.indexOf(58); + if (var2 < 0) { + throw new IllegalArgumentException(var1 + " doesn't contain encryption algorithm and password"); + } else { + String var3 = var1.substring(0, var2); + var1 = var1.substring(var2 + 1); + return new String[]{var3, var1}; + } + } + } + + public static byte[] getPasswordBytes(char[] var0) { + int var1 = var0.length; + byte[] var2 = new byte[var1 * 2]; + + for(int var3 = 0; var3 < var1; ++var3) { + char var4 = var0[var3]; + var2[var3 + var3] = (byte)(var4 >>> 8); + var2[var3 + var3 + 1] = (byte)var4; + } + + return var2; + } + + static class XTS { + private static final int GF_128_FEEDBACK = 135; + private static final int CIPHER_BLOCK_SIZE = 16; + private final BlockCipher cipher; + + XTS(BlockCipher var1) { + this.cipher = var1; + } + + void encrypt(long var1, int var3, byte[] var4, int var5) { + byte[] var6 = this.initTweak(var1); + + int var7; + for(var7 = 0; var7 + 16 <= var3; var7 += 16) { + if (var7 > 0) { + updateTweak(var6); + } + + xorTweak(var4, var7 + var5, var6); + this.cipher.encrypt(var4, var7 + var5, 16); + xorTweak(var4, var7 + var5, var6); + } + + if (var7 < var3) { + updateTweak(var6); + swap(var4, var7 + var5, var7 - 16 + var5, var3 - var7); + xorTweak(var4, var7 - 16 + var5, var6); + this.cipher.encrypt(var4, var7 - 16 + var5, 16); + xorTweak(var4, var7 - 16 + var5, var6); + } + + } + + void decrypt(long var1, int var3, byte[] var4, int var5) { + byte[] var6 = this.initTweak(var1); + byte[] var7 = var6; + + int var8; + for(var8 = 0; var8 + 16 <= var3; var8 += 16) { + if (var8 > 0) { + updateTweak(var6); + if (var8 + 16 + 16 > var3 && var8 + 16 < var3) { + var7 = (byte[])var6.clone(); + updateTweak(var6); + } + } + + xorTweak(var4, var8 + var5, var6); + this.cipher.decrypt(var4, var8 + var5, 16); + xorTweak(var4, var8 + var5, var6); + } + + if (var8 < var3) { + swap(var4, var8, var8 - 16 + var5, var3 - var8 + var5); + xorTweak(var4, var8 - 16 + var5, var7); + this.cipher.decrypt(var4, var8 - 16 + var5, 16); + xorTweak(var4, var8 - 16 + var5, var7); + } + + } + + private byte[] initTweak(long var1) { + byte[] var3 = new byte[16]; + + for(int var4 = 0; var4 < 16; var1 >>>= 8) { + var3[var4] = (byte)((int)(var1 & 255L)); + ++var4; + } + + this.cipher.encrypt(var3, 0, 16); + return var3; + } + + private static void xorTweak(byte[] var0, int var1, byte[] var2) { + for(int var3 = 0; var3 < 16; ++var3) { + var0[var1 + var3] ^= var2[var3]; + } + + } + + private static void updateTweak(byte[] var0) { + byte var1 = 0; + byte var2 = 0; + + for(int var3 = 0; var3 < 16; ++var3) { + var2 = (byte)(var0[var3] >> 7 & 1); + var0[var3] = (byte)((var0[var3] << 1) + var1 & 255); + var1 = var2; + } + + if (var2 != 0) { + var0[0] = (byte)(var0[0] ^ 135); + } + + } + + private static void swap(byte[] var0, int var1, int var2, int var3) { + for(int var4 = 0; var4 < var3; ++var4) { + byte var5 = var0[var1 + var4]; + var0[var1 + var4] = var0[var2 + var4]; + var0[var2 + var4] = var5; + } + + } + } + + public static class FileEncrypt extends FileBase { + static final int BLOCK_SIZE = 4096; + static final int BLOCK_SIZE_MASK = 4095; + static final int HEADER_LENGTH = 4096; + private static final byte[] HEADER = "H2encrypt\n".getBytes(); + private static final int SALT_POS; + private static final int SALT_LENGTH = 8; + private static final int HASH_ITERATIONS = 10; + private final FileChannel base; + private long pos; + private long size; + private final String name; + private XTS xts; + private byte[] encryptionKey; + + public FileEncrypt(String var1, byte[] var2, FileChannel var3) { + this.name = var1; + this.base = var3; + this.encryptionKey = var2; + } + + private void init() throws IOException { + if (this.xts == null) { + this.size = this.base.size() - 4096L; + boolean var1 = this.size < 0L; + byte[] var2; + if (var1) { + byte[] var3 = Arrays.copyOf(HEADER, 4096); + var2 = MathUtils.secureRandomBytes(8); + System.arraycopy(var2, 0, var3, SALT_POS, var2.length); + writeFully(this.base, 0L, ByteBuffer.wrap(var3)); + this.size = 0L; + } else { + var2 = new byte[8]; + readFully(this.base, (long)SALT_POS, ByteBuffer.wrap(var2)); + if ((this.size & 4095L) != 0L) { + this.size -= 4096L; + } + } + + AES var4 = new AES(); + var4.setKey(SHA256.getPBKDF2(this.encryptionKey, var2, 10, 16)); + this.encryptionKey = null; + this.xts = new XTS(var4); + } + } + + protected void implCloseChannel() throws IOException { + this.base.close(); + } + + public FileChannel position(long var1) throws IOException { + this.pos = var1; + return this; + } + + public long position() throws IOException { + return this.pos; + } + + public int read(ByteBuffer var1) throws IOException { + int var2 = this.read(var1, this.pos); + if (var2 > 0) { + this.pos += (long)var2; + } + + return var2; + } + + public int read(ByteBuffer var1, long var2) throws IOException { + int var4 = var1.remaining(); + if (var4 == 0) { + return 0; + } else { + this.init(); + var4 = (int)Math.min((long)var4, this.size - var2); + if (var2 >= this.size) { + return -1; + } else if (var2 < 0L) { + throw new IllegalArgumentException("pos: " + var2); + } else if ((var2 & 4095L) == 0L && (var4 & 4095) == 0) { + this.readInternal(var1, var2, var4); + return var4; + } else { + long var5 = var2 / 4096L * 4096L; + int var7 = (int)(var2 - var5); + int var8 = (var4 + var7 + 4096 - 1) / 4096 * 4096; + ByteBuffer var9 = ByteBuffer.allocate(var8); + this.readInternal(var9, var5, var8); + var9.flip(); + var9.limit(var7 + var4); + var9.position(var7); + var1.put(var9); + return var4; + } + } + } + + private void readInternal(ByteBuffer var1, long var2, int var4) throws IOException { + int var5 = var1.position(); + readFully(this.base, var2 + 4096L, var1); + + for(long var6 = var2 / 4096L; var4 > 0; var4 -= 4096) { + this.xts.decrypt(var6++, 4096, var1.array(), var1.arrayOffset() + var5); + var5 += 4096; + } + + } + + private static void readFully(FileChannel var0, long var1, ByteBuffer var3) throws IOException { + do { + int var4 = var0.read(var3, var1); + if (var4 < 0) { + throw new EOFException(); + } + + var1 += (long)var4; + } while(var3.remaining() > 0); + + } + + public int write(ByteBuffer var1, long var2) throws IOException { + this.init(); + int var4 = var1.remaining(); + long var5; + if ((var2 & 4095L) == 0L && (var4 & 4095) == 0) { + this.writeInternal(var1, var2, var4); + var5 = var2 + (long)var4; + this.size = Math.max(this.size, var5); + return var4; + } else { + var5 = var2 / 4096L * 4096L; + int var7 = (int)(var2 - var5); + int var8 = (var4 + var7 + 4096 - 1) / 4096 * 4096; + ByteBuffer var9 = ByteBuffer.allocate(var8); + int var10 = (int)(this.size - var5 + 4096L - 1L) / 4096 * 4096; + int var11 = Math.min(var8, var10); + if (var11 > 0) { + this.readInternal(var9, var5, var11); + var9.rewind(); + } + + var9.limit(var7 + var4); + var9.position(var7); + var9.put(var1); + var9.limit(var8); + var9.rewind(); + this.writeInternal(var9, var5, var8); + long var12 = var2 + (long)var4; + this.size = Math.max(this.size, var12); + int var14 = (int)(this.size & 4095L); + if (var14 > 0) { + var9 = ByteBuffer.allocate(var14); + writeFully(this.base, var5 + 4096L + (long)var8, var9); + } + + return var4; + } + } + + private void writeInternal(ByteBuffer var1, long var2, int var4) throws IOException { + ByteBuffer var5 = ByteBuffer.allocate(var4); + var5.put(var1); + var5.flip(); + long var6 = var2 / 4096L; + int var8 = 0; + + for(int var9 = var4; var9 > 0; var9 -= 4096) { + this.xts.encrypt(var6++, 4096, var5.array(), var5.arrayOffset() + var8); + var8 += 4096; + } + + writeFully(this.base, var2 + 4096L, var5); + } + + private static void writeFully(FileChannel var0, long var1, ByteBuffer var3) throws IOException { + int var4 = 0; + + do { + int var5 = var0.write(var3, var1 + (long)var4); + var4 += var5; + } while(var3.remaining() > 0); + + } + + public int write(ByteBuffer var1) throws IOException { + int var2 = this.write(var1, this.pos); + if (var2 > 0) { + this.pos += (long)var2; + } + + return var2; + } + + public long size() throws IOException { + this.init(); + return this.size; + } + + public FileChannel truncate(long var1) throws IOException { + this.init(); + if (var1 > this.size) { + return this; + } else if (var1 < 0L) { + throw new IllegalArgumentException("newSize: " + var1); + } else { + int var3 = (int)(var1 & 4095L); + if (var3 > 0) { + this.base.truncate(var1 + 4096L + 4096L); + } else { + this.base.truncate(var1 + 4096L); + } + + this.size = var1; + this.pos = Math.min(this.pos, this.size); + return this; + } + } + + public void force(boolean var1) throws IOException { + this.base.force(var1); + } + + public FileLock tryLock(long var1, long var3, boolean var5) throws IOException { + return this.base.tryLock(var1, var3, var5); + } + + public String toString() { + return this.name; + } + + static { + SALT_POS = HEADER.length; + } + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathNio.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathNio.java new file mode 100644 index 000000000..c5a512ab3 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathNio.java @@ -0,0 +1,22 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import java.io.IOException; +import java.nio.channels.FileChannel; + +public class FilePathNio extends FilePathWrapper { + public FilePathNio() { + } + + public FileChannel open(String var1) throws IOException { + return new FileNio(this.name.substring(this.getScheme().length() + 1), var1); + } + + public String getScheme() { + return "nio"; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathWrapper.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathWrapper.java new file mode 100644 index 000000000..a64627752 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FilePathWrapper.java @@ -0,0 +1,133 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; +import java.util.List; + +public abstract class FilePathWrapper extends FilePath { + private FilePath base; + + public FilePathWrapper() { + } + + public FilePathWrapper getPath(String var1) { + return this.create(var1, this.unwrap(var1)); + } + + public FilePathWrapper wrap(FilePath var1) { + return var1 == null ? null : this.create(this.getPrefix() + var1.name, var1); + } + + public FilePath unwrap() { + return this.unwrap(this.name); + } + + private FilePathWrapper create(String var1, FilePath var2) { + try { + FilePathWrapper var3 = (FilePathWrapper)this.getClass().getDeclaredConstructor().newInstance(); + var3.name = var1; + var3.base = var2; + return var3; + } catch (Exception var4) { + throw new IllegalArgumentException("Path: " + var1, var4); + } + } + + protected String getPrefix() { + return this.getScheme() + ":"; + } + + protected FilePath unwrap(String var1) { + return FilePath.get(var1.substring(this.getScheme().length() + 1)); + } + + protected FilePath getBase() { + return this.base; + } + + public boolean canWrite() { + return this.base.canWrite(); + } + + public void createDirectory() { + this.base.createDirectory(); + } + + public boolean createFile() { + return this.base.createFile(); + } + + public void delete() { + this.base.delete(); + } + + public boolean exists() { + return this.base.exists(); + } + + public FilePath getParent() { + return this.wrap(this.base.getParent()); + } + + public boolean isAbsolute() { + return this.base.isAbsolute(); + } + + public boolean isDirectory() { + return this.base.isDirectory(); + } + + public long lastModified() { + return this.base.lastModified(); + } + + public FilePath toRealPath() { + return this.wrap(this.base.toRealPath()); + } + + public List newDirectoryStream() { + List var1 = this.base.newDirectoryStream(); + int var2 = 0; + + for(int var3 = var1.size(); var2 < var3; ++var2) { + var1.set(var2, this.wrap((FilePath)var1.get(var2))); + } + + return var1; + } + + public void moveTo(FilePath var1, boolean var2) { + this.base.moveTo(((FilePathWrapper)var1).base, var2); + } + + public InputStream newInputStream() throws IOException { + return this.base.newInputStream(); + } + + public OutputStream newOutputStream(boolean var1) throws IOException { + return this.base.newOutputStream(var1); + } + + public FileChannel open(String var1) throws IOException { + return this.base.open(var1); + } + + public boolean setReadOnly() { + return this.base.setReadOnly(); + } + + public long size() { + return this.base.size(); + } + + public FilePath createTempFile(String var1, boolean var2) throws IOException { + return this.wrap(this.base.createTempFile(var1, var2)); + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileUtils.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileUtils.java new file mode 100644 index 000000000..9d9a98df3 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/fs/FileUtils.java @@ -0,0 +1,178 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.fs; + +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class FileUtils { + public FileUtils() { + } + + public static boolean exists(String var0) { + return FilePath.get(var0).exists(); + } + + public static void createDirectory(String var0) { + FilePath.get(var0).createDirectory(); + } + + public static boolean createFile(String var0) { + return FilePath.get(var0).createFile(); + } + + public static void delete(String var0) { + FilePath.get(var0).delete(); + } + + public static String toRealPath(String var0) { + return FilePath.get(var0).toRealPath().toString(); + } + + public static String getParent(String var0) { + FilePath var1 = FilePath.get(var0).getParent(); + return var1 == null ? null : var1.toString(); + } + + public static boolean isAbsolute(String var0) { + return FilePath.get(var0).isAbsolute() || var0.startsWith(File.pathSeparator) || var0.startsWith("/"); + } + + public static void move(String var0, String var1) { + FilePath.get(var0).moveTo(FilePath.get(var1), false); + } + + public static void moveAtomicReplace(String var0, String var1) { + FilePath.get(var0).moveTo(FilePath.get(var1), true); + } + + public static String getName(String var0) { + return FilePath.get(var0).getName(); + } + + public static List newDirectoryStream(String var0) { + List var1 = FilePath.get(var0).newDirectoryStream(); + int var2 = var1.size(); + ArrayList var3 = new ArrayList(var2); + Iterator var4 = var1.iterator(); + + while(var4.hasNext()) { + FilePath var5 = (FilePath)var4.next(); + var3.add(var5.toString()); + } + + return var3; + } + + public static long lastModified(String var0) { + return FilePath.get(var0).lastModified(); + } + + public static long size(String var0) { + return FilePath.get(var0).size(); + } + + public static boolean isDirectory(String var0) { + return FilePath.get(var0).isDirectory(); + } + + public static FileChannel open(String var0, String var1) throws IOException { + return FilePath.get(var0).open(var1); + } + + public static InputStream newInputStream(String var0) throws IOException { + return FilePath.get(var0).newInputStream(); + } + + public static OutputStream newOutputStream(String var0, boolean var1) throws IOException { + return FilePath.get(var0).newOutputStream(var1); + } + + public static boolean canWrite(String var0) { + return FilePath.get(var0).canWrite(); + } + + public static boolean setReadOnly(String var0) { + return FilePath.get(var0).setReadOnly(); + } + + public static String unwrap(String var0) { + return FilePath.get(var0).unwrap().toString(); + } + + public static void deleteRecursive(String var0, boolean var1) { + if (exists(var0)) { + if (isDirectory(var0)) { + Iterator var2 = newDirectoryStream(var0).iterator(); + + while(var2.hasNext()) { + String var3 = (String)var2.next(); + deleteRecursive(var3, var1); + } + } + + if (var1) { + tryDelete(var0); + } else { + delete(var0); + } + } + + } + + public static void createDirectories(String var0) { + if (var0 != null) { + if (exists(var0)) { + if (!isDirectory(var0)) { + createDirectory(var0); + } + } else { + String var1 = getParent(var0); + createDirectories(var1); + createDirectory(var0); + } + } + + } + + public static boolean tryDelete(String var0) { + try { + FilePath.get(var0).delete(); + return true; + } catch (Exception var2) { + return false; + } + } + + public static String createTempFile(String var0, String var1, boolean var2) throws IOException { + return FilePath.get(var0).createTempFile(var1, var2).toString(); + } + + public static void readFully(FileChannel var0, ByteBuffer var1) throws IOException { + do { + int var2 = var0.read(var1); + if (var2 < 0) { + throw new EOFException(); + } + } while(var1.remaining() > 0); + + } + + public static void writeFully(FileChannel var0, ByteBuffer var1) throws IOException { + do { + var0.write(var1); + } while(var1.remaining() > 0); + + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/DataType.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/DataType.java new file mode 100644 index 000000000..35eca916e --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/DataType.java @@ -0,0 +1,72 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.type; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.WriteBuffer; + +import java.nio.ByteBuffer; + +/** + * A data type. + */ +public interface DataType { + + /** + * Compare two keys. + * + * @param a the first key + * @param b the second key + * @return -1 if the first key is smaller, 1 if larger, and 0 if equal + * @throws UnsupportedOperationException if the type is not orderable + */ + int compare(Object a, Object b); + + /** + * Estimate the used memory in bytes. + * + * @param obj the object + * @return the used memory + */ + int getMemory(Object obj); + + /** + * Write an object. + * + * @param buff the target buffer + * @param obj the value + */ + void write(WriteBuffer buff, Object obj); + + /** + * Write a list of objects. + * + * @param buff the target buffer + * @param obj the objects + * @param len the number of objects to write + * @param key whether the objects are keys + */ + void write(WriteBuffer buff, Object[] obj, int len, boolean key); + + /** + * Read an object. + * + * @param buff the source buffer + * @return the object + */ + Object read(ByteBuffer buff); + + /** + * Read a list of objects. + * + * @param buff the target buffer + * @param obj the objects + * @param len the number of objects to read + * @param key whether the objects are keys + */ + void read(ByteBuffer buff, Object[] obj, int len, boolean key); + +} + diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/ObjectDataType.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/ObjectDataType.java new file mode 100644 index 000000000..ae8ee4401 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/ObjectDataType.java @@ -0,0 +1,1552 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.type; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.DataUtils; +import org.dizitart.no2.mvstore.compat.v1.mvstore.WriteBuffer; +import org.h2.util.Utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.reflect.Array; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.UUID; + +/** + * A data type implementation for the most common data types, including + * serializable objects. + */ +public class ObjectDataType implements DataType { + + /** + * The type constants are also used as tag values. + */ + static final int TYPE_NULL = 0; + static final int TYPE_BOOLEAN = 1; + static final int TYPE_BYTE = 2; + static final int TYPE_SHORT = 3; + static final int TYPE_INT = 4; + static final int TYPE_LONG = 5; + static final int TYPE_BIG_INTEGER = 6; + static final int TYPE_FLOAT = 7; + static final int TYPE_DOUBLE = 8; + static final int TYPE_BIG_DECIMAL = 9; + static final int TYPE_CHAR = 10; + static final int TYPE_STRING = 11; + static final int TYPE_UUID = 12; + static final int TYPE_DATE = 13; + static final int TYPE_ARRAY = 14; + static final int TYPE_SERIALIZED_OBJECT = 19; + + /** + * For very common values (e.g. 0 and 1) we save space by encoding the value + * in the tag. e.g. TAG_BOOLEAN_TRUE and TAG_FLOAT_0. + */ + static final int TAG_BOOLEAN_TRUE = 32; + static final int TAG_INTEGER_NEGATIVE = 33; + static final int TAG_INTEGER_FIXED = 34; + static final int TAG_LONG_NEGATIVE = 35; + static final int TAG_LONG_FIXED = 36; + static final int TAG_BIG_INTEGER_0 = 37; + static final int TAG_BIG_INTEGER_1 = 38; + static final int TAG_BIG_INTEGER_SMALL = 39; + static final int TAG_FLOAT_0 = 40; + static final int TAG_FLOAT_1 = 41; + static final int TAG_FLOAT_FIXED = 42; + static final int TAG_DOUBLE_0 = 43; + static final int TAG_DOUBLE_1 = 44; + static final int TAG_DOUBLE_FIXED = 45; + static final int TAG_BIG_DECIMAL_0 = 46; + static final int TAG_BIG_DECIMAL_1 = 47; + static final int TAG_BIG_DECIMAL_SMALL = 48; + static final int TAG_BIG_DECIMAL_SMALL_SCALED = 49; + + /** + * For small-values/small-arrays, we encode the value/array-length in the + * tag. + */ + static final int TAG_INTEGER_0_15 = 64; + static final int TAG_LONG_0_7 = 80; + static final int TAG_STRING_0_15 = 88; + static final int TAG_BYTE_ARRAY_0_15 = 104; + + /** + * Constants for floating point synchronization. + */ + static final int FLOAT_ZERO_BITS = Float.floatToIntBits(0.0f); + static final int FLOAT_ONE_BITS = Float.floatToIntBits(1.0f); + static final long DOUBLE_ZERO_BITS = Double.doubleToLongBits(0.0d); + static final long DOUBLE_ONE_BITS = Double.doubleToLongBits(1.0d); + + static final Class[] COMMON_CLASSES = { boolean.class, byte.class, + short.class, char.class, int.class, long.class, float.class, + double.class, Object.class, Boolean.class, Byte.class, Short.class, + Character.class, Integer.class, Long.class, BigInteger.class, + Float.class, Double.class, BigDecimal.class, String.class, + UUID.class, Date.class }; + + private static final HashMap, Integer> COMMON_CLASSES_MAP = new HashMap<>(32); + + private AutoDetectDataType last = new StringType(this); + + @Override + public int compare(Object a, Object b) { + return last.compare(a, b); + } + + @Override + public int getMemory(Object obj) { + return last.getMemory(obj); + } + + @Override + public void read(ByteBuffer buff, Object[] obj, int len, boolean key) { + for (int i = 0; i < len; i++) { + obj[i] = read(buff); + } + } + + @Override + public void write(WriteBuffer buff, Object[] obj, int len, boolean key) { + for (int i = 0; i < len; i++) { + write(buff, obj[i]); + } + } + + @Override + public void write(WriteBuffer buff, Object obj) { + last.write(buff, obj); + } + + private AutoDetectDataType newType(int typeId) { + switch (typeId) { + case TYPE_NULL: + return new NullType(this); + case TYPE_BOOLEAN: + return new BooleanType(this); + case TYPE_BYTE: + return new ByteType(this); + case TYPE_SHORT: + return new ShortType(this); + case TYPE_CHAR: + return new CharacterType(this); + case TYPE_INT: + return new IntegerType(this); + case TYPE_LONG: + return new LongType(this); + case TYPE_FLOAT: + return new FloatType(this); + case TYPE_DOUBLE: + return new DoubleType(this); + case TYPE_BIG_INTEGER: + return new BigIntegerType(this); + case TYPE_BIG_DECIMAL: + return new BigDecimalType(this); + case TYPE_STRING: + return new StringType(this); + case TYPE_UUID: + return new UUIDType(this); + case TYPE_DATE: + return new DateType(this); + case TYPE_ARRAY: + return new ObjectArrayType(this); + case TYPE_SERIALIZED_OBJECT: + return new SerializedObjectType(this); + } + throw DataUtils.newIllegalStateException(DataUtils.ERROR_INTERNAL, + "Unsupported type {0}", typeId); + } + + @Override + public Object read(ByteBuffer buff) { + int tag = buff.get(); + int typeId; + if (tag <= TYPE_SERIALIZED_OBJECT) { + typeId = tag; + } else { + switch (tag) { + case TAG_BOOLEAN_TRUE: + typeId = TYPE_BOOLEAN; + break; + case TAG_INTEGER_NEGATIVE: + case TAG_INTEGER_FIXED: + typeId = TYPE_INT; + break; + case TAG_LONG_NEGATIVE: + case TAG_LONG_FIXED: + typeId = TYPE_LONG; + break; + case TAG_BIG_INTEGER_0: + case TAG_BIG_INTEGER_1: + case TAG_BIG_INTEGER_SMALL: + typeId = TYPE_BIG_INTEGER; + break; + case TAG_FLOAT_0: + case TAG_FLOAT_1: + case TAG_FLOAT_FIXED: + typeId = TYPE_FLOAT; + break; + case TAG_DOUBLE_0: + case TAG_DOUBLE_1: + case TAG_DOUBLE_FIXED: + typeId = TYPE_DOUBLE; + break; + case TAG_BIG_DECIMAL_0: + case TAG_BIG_DECIMAL_1: + case TAG_BIG_DECIMAL_SMALL: + case TAG_BIG_DECIMAL_SMALL_SCALED: + typeId = TYPE_BIG_DECIMAL; + break; + default: + if (tag >= TAG_INTEGER_0_15 && tag <= TAG_INTEGER_0_15 + 15) { + typeId = TYPE_INT; + } else if (tag >= TAG_STRING_0_15 + && tag <= TAG_STRING_0_15 + 15) { + typeId = TYPE_STRING; + } else if (tag >= TAG_LONG_0_7 && tag <= TAG_LONG_0_7 + 7) { + typeId = TYPE_LONG; + } else if (tag >= TAG_BYTE_ARRAY_0_15 + && tag <= TAG_BYTE_ARRAY_0_15 + 15) { + typeId = TYPE_ARRAY; + } else { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_FILE_CORRUPT, "Unknown tag {0}", + tag); + } + } + } + AutoDetectDataType t = last; + if (typeId != t.typeId) { + last = t = newType(typeId); + } + return t.read(buff, tag); + } + + private static int getTypeId(Object obj) { + if (obj instanceof Integer) { + return TYPE_INT; + } else if (obj instanceof String) { + return TYPE_STRING; + } else if (obj instanceof Long) { + return TYPE_LONG; + } else if (obj instanceof Double) { + return TYPE_DOUBLE; + } else if (obj instanceof Float) { + return TYPE_FLOAT; + } else if (obj instanceof Boolean) { + return TYPE_BOOLEAN; + } else if (obj instanceof UUID) { + return TYPE_UUID; + } else if (obj instanceof Byte) { + return TYPE_BYTE; + } else if (obj instanceof Short) { + return TYPE_SHORT; + } else if (obj instanceof Character) { + return TYPE_CHAR; + } else if (obj == null) { + return TYPE_NULL; + } else if (isDate(obj)) { + return TYPE_DATE; + } else if (isBigInteger(obj)) { + return TYPE_BIG_INTEGER; + } else if (isBigDecimal(obj)) { + return TYPE_BIG_DECIMAL; + } else if (obj.getClass().isArray()) { + return TYPE_ARRAY; + } + return TYPE_SERIALIZED_OBJECT; + } + + /** + * Switch the last remembered type to match the type of the given object. + * + * @param obj the object + * @return the auto-detected type used + */ + AutoDetectDataType switchType(Object obj) { + int typeId = getTypeId(obj); + AutoDetectDataType l = last; + if (typeId != l.typeId) { + last = l = newType(typeId); + } + return l; + } + + /** + * Check whether this object is a BigInteger. + * + * @param obj the object + * @return true if yes + */ + static boolean isBigInteger(Object obj) { + return obj != null && obj.getClass() == BigInteger.class; + } + + /** + * Check whether this object is a BigDecimal. + * + * @param obj the object + * @return true if yes + */ + static boolean isBigDecimal(Object obj) { + return obj != null && obj.getClass() == BigDecimal.class; + } + + /** + * Check whether this object is a date. + * + * @param obj the object + * @return true if yes + */ + static boolean isDate(Object obj) { + return obj != null && obj.getClass() == Date.class; + } + + /** + * Check whether this object is an array. + * + * @param obj the object + * @return true if yes + */ + static boolean isArray(Object obj) { + return obj != null && obj.getClass().isArray(); + } + + /** + * Get the class id, or null if not found. + * + * @param clazz the class + * @return the class id or null + */ + static Integer getCommonClassId(Class clazz) { + HashMap, Integer> map = COMMON_CLASSES_MAP; + if (map.size() == 0) { + // lazy initialization + // synchronized, because the COMMON_CLASSES_MAP is not + synchronized (map) { + if (map.size() == 0) { + for (int i = 0, size = COMMON_CLASSES.length; i < size; i++) { + map.put(COMMON_CLASSES[i], i); + } + } + } + } + return map.get(clazz); + } + + /** + * Serialize the object to a byte array. + * + * @param obj the object to serialize + * @return the byte array + */ + public static byte[] serialize(Object obj) { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ObjectOutputStream os = new ObjectOutputStream(out); + os.writeObject(obj); + return out.toByteArray(); + } catch (Throwable e) { + throw DataUtils.newIllegalArgumentException( + "Could not serialize {0}", obj, e); + } + } + + /** + * De-serialize the byte array to an object. + * + * @param data the byte array + * @return the object + */ + public static Object deserialize(byte[] data) { + try { + ByteArrayInputStream in = new ByteArrayInputStream(data); + ObjectInputStream is = new ObjectInputStream(in); + return is.readObject(); + } catch (Throwable e) { + throw DataUtils.newIllegalArgumentException( + "Could not deserialize {0}", Arrays.toString(data), e); + } + } + + /** + * Compare the contents of two byte arrays. If the content or length of the + * first array is smaller than the second array, -1 is returned. If the + * content or length of the second array is smaller than the first array, 1 + * is returned. If the contents and lengths are the same, 0 is returned. + *

+ * This method interprets bytes as unsigned. + * + * @param data1 the first byte array (must not be null) + * @param data2 the second byte array (must not be null) + * @return the result of the comparison (-1, 1 or 0) + */ + public static int compareNotNull(byte[] data1, byte[] data2) { + if (data1 == data2) { + return 0; + } + int len = Math.min(data1.length, data2.length); + for (int i = 0; i < len; i++) { + int b = data1[i] & 255; + int b2 = data2[i] & 255; + if (b != b2) { + return b > b2 ? 1 : -1; + } + } + return Integer.signum(data1.length - data2.length); + } + + /** + * The base class for auto-detect data types. + */ + abstract static class AutoDetectDataType implements DataType { + + protected final ObjectDataType base; + protected final int typeId; + + AutoDetectDataType(ObjectDataType base, int typeId) { + this.base = base; + this.typeId = typeId; + } + + @Override + public int getMemory(Object o) { + return getType(o).getMemory(o); + } + + @Override + public int compare(Object aObj, Object bObj) { + AutoDetectDataType aType = getType(aObj); + AutoDetectDataType bType = getType(bObj); + int typeDiff = aType.typeId - bType.typeId; + if (typeDiff == 0) { + return aType.compare(aObj, bObj); + } + return Integer.signum(typeDiff); + } + + @Override + public void write(WriteBuffer buff, Object[] obj, + int len, boolean key) { + for (int i = 0; i < len; i++) { + write(buff, obj[i]); + } + } + + @Override + public void write(WriteBuffer buff, Object o) { + getType(o).write(buff, o); + } + + @Override + public void read(ByteBuffer buff, Object[] obj, + int len, boolean key) { + for (int i = 0; i < len; i++) { + obj[i] = read(buff); + } + } + + @Override + public final Object read(ByteBuffer buff) { + throw DataUtils.newIllegalStateException(DataUtils.ERROR_INTERNAL, + "Internal error"); + } + + /** + * Get the type for the given object. + * + * @param o the object + * @return the type + */ + AutoDetectDataType getType(Object o) { + return base.switchType(o); + } + + /** + * Read an object from the buffer. + * + * @param buff the buffer + * @param tag the first byte of the object (usually the type) + * @return the read object + */ + abstract Object read(ByteBuffer buff, int tag); + + } + + /** + * The type for the null value + */ + static class NullType extends AutoDetectDataType { + + NullType(ObjectDataType base) { + super(base, TYPE_NULL); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj == null && bObj == null) { + return 0; + } else if (aObj == null) { + return -1; + } else if (bObj == null) { + return 1; + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj == null ? 0 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (obj != null) { + super.write(buff, obj); + return; + } + buff.put((byte) TYPE_NULL); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + return null; + } + + } + + /** + * The type for boolean true and false. + */ + static class BooleanType extends AutoDetectDataType { + + BooleanType(ObjectDataType base) { + super(base, TYPE_BOOLEAN); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Boolean && bObj instanceof Boolean) { + Boolean a = (Boolean) aObj; + Boolean b = (Boolean) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Boolean ? 0 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Boolean)) { + super.write(buff, obj); + return; + } + int tag = ((Boolean) obj) ? TAG_BOOLEAN_TRUE : TYPE_BOOLEAN; + buff.put((byte) tag); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + return tag == TYPE_BOOLEAN ? Boolean.FALSE : Boolean.TRUE; + } + + } + + /** + * The type for byte objects. + */ + static class ByteType extends AutoDetectDataType { + + ByteType(ObjectDataType base) { + super(base, TYPE_BYTE); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Byte && bObj instanceof Byte) { + Byte a = (Byte) aObj; + Byte b = (Byte) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Byte ? 0 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Byte)) { + super.write(buff, obj); + return; + } + buff.put((byte) TYPE_BYTE); + buff.put((Byte) obj); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + return buff.get(); + } + + } + + /** + * The type for character objects. + */ + static class CharacterType extends AutoDetectDataType { + + CharacterType(ObjectDataType base) { + super(base, TYPE_CHAR); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Character && bObj instanceof Character) { + Character a = (Character) aObj; + Character b = (Character) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Character ? 24 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Character)) { + super.write(buff, obj); + return; + } + buff.put((byte) TYPE_CHAR); + buff.putChar((Character) obj); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + return buff.getChar(); + } + + } + + /** + * The type for short objects. + */ + static class ShortType extends AutoDetectDataType { + + ShortType(ObjectDataType base) { + super(base, TYPE_SHORT); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Short && bObj instanceof Short) { + Short a = (Short) aObj; + Short b = (Short) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Short ? 24 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Short)) { + super.write(buff, obj); + return; + } + buff.put((byte) TYPE_SHORT); + buff.putShort((Short) obj); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + return buff.getShort(); + } + + } + + /** + * The type for integer objects. + */ + static class IntegerType extends AutoDetectDataType { + + IntegerType(ObjectDataType base) { + super(base, TYPE_INT); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Integer && bObj instanceof Integer) { + Integer a = (Integer) aObj; + Integer b = (Integer) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Integer ? 24 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Integer)) { + super.write(buff, obj); + return; + } + int x = (Integer) obj; + if (x < 0) { + // -Integer.MIN_VALUE is smaller than 0 + if (-x < 0 || -x > DataUtils.COMPRESSED_VAR_INT_MAX) { + buff.put((byte) TAG_INTEGER_FIXED).putInt(x); + } else { + buff.put((byte) TAG_INTEGER_NEGATIVE).putVarInt(-x); + } + } else if (x <= 15) { + buff.put((byte) (TAG_INTEGER_0_15 + x)); + } else if (x <= DataUtils.COMPRESSED_VAR_INT_MAX) { + buff.put((byte) TYPE_INT).putVarInt(x); + } else { + buff.put((byte) TAG_INTEGER_FIXED).putInt(x); + } + } + + @Override + public Object read(ByteBuffer buff, int tag) { + switch (tag) { + case TYPE_INT: + return DataUtils.readVarInt(buff); + case TAG_INTEGER_NEGATIVE: + return -DataUtils.readVarInt(buff); + case TAG_INTEGER_FIXED: + return buff.getInt(); + } + return tag - TAG_INTEGER_0_15; + } + + } + + /** + * The type for long objects. + */ + static class LongType extends AutoDetectDataType { + + LongType(ObjectDataType base) { + super(base, TYPE_LONG); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Long && bObj instanceof Long) { + Long a = (Long) aObj; + Long b = (Long) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Long ? 30 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Long)) { + super.write(buff, obj); + return; + } + long x = (Long) obj; + if (x < 0) { + // -Long.MIN_VALUE is smaller than 0 + if (-x < 0 || -x > DataUtils.COMPRESSED_VAR_LONG_MAX) { + buff.put((byte) TAG_LONG_FIXED); + buff.putLong(x); + } else { + buff.put((byte) TAG_LONG_NEGATIVE); + buff.putVarLong(-x); + } + } else if (x <= 7) { + buff.put((byte) (TAG_LONG_0_7 + x)); + } else if (x <= DataUtils.COMPRESSED_VAR_LONG_MAX) { + buff.put((byte) TYPE_LONG); + buff.putVarLong(x); + } else { + buff.put((byte) TAG_LONG_FIXED); + buff.putLong(x); + } + } + + @Override + public Object read(ByteBuffer buff, int tag) { + switch (tag) { + case TYPE_LONG: + return DataUtils.readVarLong(buff); + case TAG_LONG_NEGATIVE: + return -DataUtils.readVarLong(buff); + case TAG_LONG_FIXED: + return buff.getLong(); + } + return (long) (tag - TAG_LONG_0_7); + } + + } + + /** + * The type for float objects. + */ + static class FloatType extends AutoDetectDataType { + + FloatType(ObjectDataType base) { + super(base, TYPE_FLOAT); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Float && bObj instanceof Float) { + Float a = (Float) aObj; + Float b = (Float) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Float ? 24 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Float)) { + super.write(buff, obj); + return; + } + float x = (Float) obj; + int f = Float.floatToIntBits(x); + if (f == ObjectDataType.FLOAT_ZERO_BITS) { + buff.put((byte) TAG_FLOAT_0); + } else if (f == ObjectDataType.FLOAT_ONE_BITS) { + buff.put((byte) TAG_FLOAT_1); + } else { + int value = Integer.reverse(f); + if (value >= 0 && value <= DataUtils.COMPRESSED_VAR_INT_MAX) { + buff.put((byte) TYPE_FLOAT).putVarInt(value); + } else { + buff.put((byte) TAG_FLOAT_FIXED).putFloat(x); + } + } + } + + @Override + public Object read(ByteBuffer buff, int tag) { + switch (tag) { + case TAG_FLOAT_0: + return 0f; + case TAG_FLOAT_1: + return 1f; + case TAG_FLOAT_FIXED: + return buff.getFloat(); + } + return Float.intBitsToFloat(Integer.reverse(DataUtils + .readVarInt(buff))); + } + + } + + /** + * The type for double objects. + */ + static class DoubleType extends AutoDetectDataType { + + DoubleType(ObjectDataType base) { + super(base, TYPE_DOUBLE); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof Double && bObj instanceof Double) { + Double a = (Double) aObj; + Double b = (Double) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof Double ? 30 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof Double)) { + super.write(buff, obj); + return; + } + double x = (Double) obj; + long d = Double.doubleToLongBits(x); + if (d == ObjectDataType.DOUBLE_ZERO_BITS) { + buff.put((byte) TAG_DOUBLE_0); + } else if (d == ObjectDataType.DOUBLE_ONE_BITS) { + buff.put((byte) TAG_DOUBLE_1); + } else { + long value = Long.reverse(d); + if (value >= 0 && value <= DataUtils.COMPRESSED_VAR_LONG_MAX) { + buff.put((byte) TYPE_DOUBLE); + buff.putVarLong(value); + } else { + buff.put((byte) TAG_DOUBLE_FIXED); + buff.putDouble(x); + } + } + } + + @Override + public Object read(ByteBuffer buff, int tag) { + switch (tag) { + case TAG_DOUBLE_0: + return 0d; + case TAG_DOUBLE_1: + return 1d; + case TAG_DOUBLE_FIXED: + return buff.getDouble(); + } + return Double.longBitsToDouble(Long.reverse(DataUtils + .readVarLong(buff))); + } + + } + + /** + * The type for BigInteger objects. + */ + static class BigIntegerType extends AutoDetectDataType { + + BigIntegerType(ObjectDataType base) { + super(base, TYPE_BIG_INTEGER); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (isBigInteger(aObj) && isBigInteger(bObj)) { + BigInteger a = (BigInteger) aObj; + BigInteger b = (BigInteger) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return isBigInteger(obj) ? 100 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!isBigInteger(obj)) { + super.write(buff, obj); + return; + } + BigInteger x = (BigInteger) obj; + if (BigInteger.ZERO.equals(x)) { + buff.put((byte) TAG_BIG_INTEGER_0); + } else if (BigInteger.ONE.equals(x)) { + buff.put((byte) TAG_BIG_INTEGER_1); + } else { + int bits = x.bitLength(); + if (bits <= 63) { + buff.put((byte) TAG_BIG_INTEGER_SMALL).putVarLong( + x.longValue()); + } else { + byte[] bytes = x.toByteArray(); + buff.put((byte) TYPE_BIG_INTEGER).putVarInt(bytes.length) + .put(bytes); + } + } + } + + @Override + public Object read(ByteBuffer buff, int tag) { + switch (tag) { + case TAG_BIG_INTEGER_0: + return BigInteger.ZERO; + case TAG_BIG_INTEGER_1: + return BigInteger.ONE; + case TAG_BIG_INTEGER_SMALL: + return BigInteger.valueOf(DataUtils.readVarLong(buff)); + } + int len = DataUtils.readVarInt(buff); + byte[] bytes = Utils.newBytes(len); + buff.get(bytes); + return new BigInteger(bytes); + } + + } + + /** + * The type for BigDecimal objects. + */ + static class BigDecimalType extends AutoDetectDataType { + + BigDecimalType(ObjectDataType base) { + super(base, TYPE_BIG_DECIMAL); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (isBigDecimal(aObj) && isBigDecimal(bObj)) { + BigDecimal a = (BigDecimal) aObj; + BigDecimal b = (BigDecimal) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public int getMemory(Object obj) { + return isBigDecimal(obj) ? 150 : super.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!isBigDecimal(obj)) { + super.write(buff, obj); + return; + } + BigDecimal x = (BigDecimal) obj; + if (BigDecimal.ZERO.equals(x)) { + buff.put((byte) TAG_BIG_DECIMAL_0); + } else if (BigDecimal.ONE.equals(x)) { + buff.put((byte) TAG_BIG_DECIMAL_1); + } else { + int scale = x.scale(); + BigInteger b = x.unscaledValue(); + int bits = b.bitLength(); + if (bits < 64) { + if (scale == 0) { + buff.put((byte) TAG_BIG_DECIMAL_SMALL); + } else { + buff.put((byte) TAG_BIG_DECIMAL_SMALL_SCALED) + .putVarInt(scale); + } + buff.putVarLong(b.longValue()); + } else { + byte[] bytes = b.toByteArray(); + buff.put((byte) TYPE_BIG_DECIMAL).putVarInt(scale) + .putVarInt(bytes.length).put(bytes); + } + } + } + + @Override + public Object read(ByteBuffer buff, int tag) { + switch (tag) { + case TAG_BIG_DECIMAL_0: + return BigDecimal.ZERO; + case TAG_BIG_DECIMAL_1: + return BigDecimal.ONE; + case TAG_BIG_DECIMAL_SMALL: + return BigDecimal.valueOf(DataUtils.readVarLong(buff)); + case TAG_BIG_DECIMAL_SMALL_SCALED: + int scale = DataUtils.readVarInt(buff); + return BigDecimal.valueOf(DataUtils.readVarLong(buff), scale); + } + int scale = DataUtils.readVarInt(buff); + int len = DataUtils.readVarInt(buff); + byte[] bytes = Utils.newBytes(len); + buff.get(bytes); + BigInteger b = new BigInteger(bytes); + return new BigDecimal(b, scale); + } + + } + + /** + * The type for string objects. + */ + static class StringType extends AutoDetectDataType { + + StringType(ObjectDataType base) { + super(base, TYPE_STRING); + } + + @Override + public int getMemory(Object obj) { + if (!(obj instanceof String)) { + return super.getMemory(obj); + } + return 24 + 2 * obj.toString().length(); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof String && bObj instanceof String) { + return aObj.toString().compareTo(bObj.toString()); + } + return super.compare(aObj, bObj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof String)) { + super.write(buff, obj); + return; + } + String s = (String) obj; + int len = s.length(); + if (len <= 15) { + buff.put((byte) (TAG_STRING_0_15 + len)); + } else { + buff.put((byte) TYPE_STRING).putVarInt(len); + } + buff.putStringData(s, len); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + int len; + if (tag == TYPE_STRING) { + len = DataUtils.readVarInt(buff); + } else { + len = tag - TAG_STRING_0_15; + } + return DataUtils.readString(buff, len); + } + + } + + /** + * The type for UUID objects. + */ + static class UUIDType extends AutoDetectDataType { + + UUIDType(ObjectDataType base) { + super(base, TYPE_UUID); + } + + @Override + public int getMemory(Object obj) { + return obj instanceof UUID ? 40 : super.getMemory(obj); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (aObj instanceof UUID && bObj instanceof UUID) { + UUID a = (UUID) aObj; + UUID b = (UUID) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!(obj instanceof UUID)) { + super.write(buff, obj); + return; + } + buff.put((byte) TYPE_UUID); + UUID a = (UUID) obj; + buff.putLong(a.getMostSignificantBits()); + buff.putLong(a.getLeastSignificantBits()); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + long a = buff.getLong(), b = buff.getLong(); + return new UUID(a, b); + } + + } + + /** + * The type for java.util.Date objects. + */ + static class DateType extends AutoDetectDataType { + + DateType(ObjectDataType base) { + super(base, TYPE_DATE); + } + + @Override + public int getMemory(Object obj) { + return isDate(obj) ? 40 : super.getMemory(obj); + } + + @Override + public int compare(Object aObj, Object bObj) { + if (isDate(aObj) && isDate(bObj)) { + Date a = (Date) aObj; + Date b = (Date) bObj; + return a.compareTo(b); + } + return super.compare(aObj, bObj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!isDate(obj)) { + super.write(buff, obj); + return; + } + buff.put((byte) TYPE_DATE); + Date a = (Date) obj; + buff.putLong(a.getTime()); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + long a = buff.getLong(); + return new Date(a); + } + + } + + /** + * The type for object arrays. + */ + static class ObjectArrayType extends AutoDetectDataType { + + private final ObjectDataType elementType = new ObjectDataType(); + + ObjectArrayType(ObjectDataType base) { + super(base, TYPE_ARRAY); + } + + @Override + public int getMemory(Object obj) { + if (!isArray(obj)) { + return super.getMemory(obj); + } + int size = 64; + Class type = obj.getClass().getComponentType(); + if (type.isPrimitive()) { + int len = Array.getLength(obj); + if (type == boolean.class || type == byte.class) { + size += len; + } else if (type == char.class || type == short.class) { + size += len * 2; + } else if (type == int.class || type == float.class) { + size += len * 4; + } else if (type == double.class || type == long.class) { + size += len * 8; + } + } else { + for (Object x : (Object[]) obj) { + if (x != null) { + size += elementType.getMemory(x); + } + } + } + // we say they are larger, because these objects + // use quite a lot of disk space + return size * 2; + } + + @Override + public int compare(Object aObj, Object bObj) { + if (!isArray(aObj) || !isArray(bObj)) { + return super.compare(aObj, bObj); + } + if (aObj == bObj) { + return 0; + } + Class type = aObj.getClass().getComponentType(); + Class bType = bObj.getClass().getComponentType(); + if (type != bType) { + Integer classA = getCommonClassId(type); + Integer classB = getCommonClassId(bType); + if (classA != null) { + if (classB != null) { + return classA.compareTo(classB); + } + return -1; + } else if (classB != null) { + return 1; + } + return type.getName().compareTo(bType.getName()); + } + int aLen = Array.getLength(aObj); + int bLen = Array.getLength(bObj); + int len = Math.min(aLen, bLen); + if (type.isPrimitive()) { + if (type == byte.class) { + byte[] a = (byte[]) aObj; + byte[] b = (byte[]) bObj; + return compareNotNull(a, b); + } + for (int i = 0; i < len; i++) { + int x; + if (type == boolean.class) { + x = Integer.signum((((boolean[]) aObj)[i] ? 1 : 0) + - (((boolean[]) bObj)[i] ? 1 : 0)); + } else if (type == char.class) { + x = Integer.signum((((char[]) aObj)[i]) + - (((char[]) bObj)[i])); + } else if (type == short.class) { + x = Integer.signum((((short[]) aObj)[i]) + - (((short[]) bObj)[i])); + } else if (type == int.class) { + int a = ((int[]) aObj)[i]; + int b = ((int[]) bObj)[i]; + x = Integer.compare(a, b); + } else if (type == float.class) { + x = Float.compare(((float[]) aObj)[i], + ((float[]) bObj)[i]); + } else if (type == double.class) { + x = Double.compare(((double[]) aObj)[i], + ((double[]) bObj)[i]); + } else { + long a = ((long[]) aObj)[i]; + long b = ((long[]) bObj)[i]; + x = Long.compare(a, b); + } + if (x != 0) { + return x; + } + } + } else { + Object[] a = (Object[]) aObj; + Object[] b = (Object[]) bObj; + for (int i = 0; i < len; i++) { + int comp = elementType.compare(a[i], b[i]); + if (comp != 0) { + return comp; + } + } + } + return Integer.compare(aLen, bLen); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + if (!isArray(obj)) { + super.write(buff, obj); + return; + } + Class type = obj.getClass().getComponentType(); + Integer classId = getCommonClassId(type); + if (classId != null) { + if (type.isPrimitive()) { + if (type == byte.class) { + byte[] data = (byte[]) obj; + int len = data.length; + if (len <= 15) { + buff.put((byte) (TAG_BYTE_ARRAY_0_15 + len)); + } else { + buff.put((byte) TYPE_ARRAY) + .put((byte) classId.intValue()) + .putVarInt(len); + } + buff.put(data); + return; + } + int len = Array.getLength(obj); + buff.put((byte) TYPE_ARRAY).put((byte) classId.intValue()) + .putVarInt(len); + for (int i = 0; i < len; i++) { + if (type == boolean.class) { + buff.put((byte) (((boolean[]) obj)[i] ? 1 : 0)); + } else if (type == char.class) { + buff.putChar(((char[]) obj)[i]); + } else if (type == short.class) { + buff.putShort(((short[]) obj)[i]); + } else if (type == int.class) { + buff.putInt(((int[]) obj)[i]); + } else if (type == float.class) { + buff.putFloat(((float[]) obj)[i]); + } else if (type == double.class) { + buff.putDouble(((double[]) obj)[i]); + } else { + buff.putLong(((long[]) obj)[i]); + } + } + return; + } + buff.put((byte) TYPE_ARRAY).put((byte) classId.intValue()); + } else { + buff.put((byte) TYPE_ARRAY).put((byte) -1); + String c = type.getName(); + StringDataType.INSTANCE.write(buff, c); + } + Object[] array = (Object[]) obj; + int len = array.length; + buff.putVarInt(len); + for (Object x : array) { + elementType.write(buff, x); + } + } + + @Override + public Object read(ByteBuffer buff, int tag) { + if (tag != TYPE_ARRAY) { + byte[] data; + int len = tag - TAG_BYTE_ARRAY_0_15; + data = Utils.newBytes(len); + buff.get(data); + return data; + } + int ct = buff.get(); + Class clazz; + Object obj; + if (ct == -1) { + String componentType = StringDataType.INSTANCE.read(buff); + try { + clazz = Class.forName(componentType); + } catch (Exception e) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_SERIALIZATION, + "Could not get class {0}", componentType, e); + } + } else { + clazz = COMMON_CLASSES[ct]; + } + int len = DataUtils.readVarInt(buff); + try { + obj = Array.newInstance(clazz, len); + } catch (Exception e) { + throw DataUtils.newIllegalStateException( + DataUtils.ERROR_SERIALIZATION, + "Could not create array of type {0} length {1}", clazz, + len, e); + } + if (clazz.isPrimitive()) { + for (int i = 0; i < len; i++) { + if (clazz == boolean.class) { + ((boolean[]) obj)[i] = buff.get() == 1; + } else if (clazz == byte.class) { + ((byte[]) obj)[i] = buff.get(); + } else if (clazz == char.class) { + ((char[]) obj)[i] = buff.getChar(); + } else if (clazz == short.class) { + ((short[]) obj)[i] = buff.getShort(); + } else if (clazz == int.class) { + ((int[]) obj)[i] = buff.getInt(); + } else if (clazz == float.class) { + ((float[]) obj)[i] = buff.getFloat(); + } else if (clazz == double.class) { + ((double[]) obj)[i] = buff.getDouble(); + } else { + ((long[]) obj)[i] = buff.getLong(); + } + } + } else { + Object[] array = (Object[]) obj; + for (int i = 0; i < len; i++) { + array[i] = elementType.read(buff); + } + } + return obj; + } + + } + + /** + * The type for serialized objects. + */ + static class SerializedObjectType extends AutoDetectDataType { + + private int averageSize = 10_000; + + SerializedObjectType(ObjectDataType base) { + super(base, TYPE_SERIALIZED_OBJECT); + } + + @SuppressWarnings("unchecked") + @Override + public int compare(Object aObj, Object bObj) { + if (aObj == bObj) { + return 0; + } + DataType ta = getType(aObj); + DataType tb = getType(bObj); + if (ta != this || tb != this) { + if (ta == tb) { + return ta.compare(aObj, bObj); + } + return super.compare(aObj, bObj); + } + // TODO ensure comparable type (both may be comparable but not + // with each other) + if (aObj instanceof Comparable) { + if (aObj.getClass().isAssignableFrom(bObj.getClass())) { + return ((Comparable) aObj).compareTo(bObj); + } + } + if (bObj instanceof Comparable) { + if (bObj.getClass().isAssignableFrom(aObj.getClass())) { + return -((Comparable) bObj).compareTo(aObj); + } + } + byte[] a = serialize(aObj); + byte[] b = serialize(bObj); + return compareNotNull(a, b); + } + + @Override + public int getMemory(Object obj) { + DataType t = getType(obj); + if (t == this) { + return averageSize; + } + return t.getMemory(obj); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + DataType t = getType(obj); + if (t != this) { + t.write(buff, obj); + return; + } + byte[] data = serialize(obj); + // we say they are larger, because these objects + // use quite a lot of disk space + int size = data.length * 2; + // adjust the average size + // using an exponential moving average + averageSize = (size + 15 * averageSize) / 16; + buff.put((byte) TYPE_SERIALIZED_OBJECT).putVarInt(data.length) + .put(data); + } + + @Override + public Object read(ByteBuffer buff, int tag) { + int len = DataUtils.readVarInt(buff); + byte[] data = Utils.newBytes(len); + int size = data.length * 2; + // adjust the average size + // using an exponential moving average + averageSize = (size + 15 * averageSize) / 16; + buff.get(data); + return deserialize(data); + } + } + +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/StringDataType.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/StringDataType.java new file mode 100644 index 000000000..4b954809d --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/type/StringDataType.java @@ -0,0 +1,57 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.type; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.DataUtils; +import org.dizitart.no2.mvstore.compat.v1.mvstore.WriteBuffer; + +import java.nio.ByteBuffer; + +/** + * A string type. + */ +public class StringDataType implements DataType { + + public static final StringDataType INSTANCE = new StringDataType(); + + @Override + public int compare(Object a, Object b) { + return a.toString().compareTo(b.toString()); + } + + @Override + public int getMemory(Object obj) { + return 24 + 2 * obj.toString().length(); + } + + @Override + public void read(ByteBuffer buff, Object[] obj, int len, boolean key) { + for (int i = 0; i < len; i++) { + obj[i] = read(buff); + } + } + + @Override + public void write(WriteBuffer buff, Object[] obj, int len, boolean key) { + for (int i = 0; i < len; i++) { + write(buff, obj[i]); + } + } + + @Override + public String read(ByteBuffer buff) { + return DataUtils.readString(buff); + } + + @Override + public void write(WriteBuffer buff, Object obj) { + String s = obj.toString(); + int len = s.length(); + buff.putVarInt(len).putStringData(s, len); + } + +} + diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/Bits.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/Bits.java new file mode 100644 index 000000000..e0e5fc38d --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/Bits.java @@ -0,0 +1,138 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ +package org.dizitart.no2.mvstore.compat.v1.mvstore.util; + +import java.util.UUID; + +public final class Bits { + public static int compareNotNull(char[] var0, char[] var1) { + if (var0 == var1) { + return 0; + } else { + int var2 = Math.min(var0.length, var1.length); + + for(int var3 = 0; var3 < var2; ++var3) { + char var4 = var0[var3]; + char var5 = var1[var3]; + if (var4 != var5) { + return var4 > var5 ? 1 : -1; + } + } + + return Integer.signum(var0.length - var1.length); + } + } + + public static int compareNotNullSigned(byte[] var0, byte[] var1) { + if (var0 == var1) { + return 0; + } else { + int var2 = Math.min(var0.length, var1.length); + + for(int var3 = 0; var3 < var2; ++var3) { + byte var4 = var0[var3]; + byte var5 = var1[var3]; + if (var4 != var5) { + return var4 > var5 ? 1 : -1; + } + } + + return Integer.signum(var0.length - var1.length); + } + } + + public static int compareNotNullUnsigned(byte[] var0, byte[] var1) { + if (var0 == var1) { + return 0; + } else { + int var2 = Math.min(var0.length, var1.length); + + for(int var3 = 0; var3 < var2; ++var3) { + int var4 = var0[var3] & 255; + int var5 = var1[var3] & 255; + if (var4 != var5) { + return var4 > var5 ? 1 : -1; + } + } + + return Integer.signum(var0.length - var1.length); + } + } + + public static int readInt(byte[] var0, int var1) { + return (var0[var1++] << 24) + ((var0[var1++] & 255) << 16) + ((var0[var1++] & 255) << 8) + (var0[var1] & 255); + } + + public static int readIntLE(byte[] var0, int var1) { + return (var0[var1++] & 255) + ((var0[var1++] & 255) << 8) + ((var0[var1++] & 255) << 16) + (var0[var1] << 24); + } + + public static long readLong(byte[] var0, int var1) { + return ((long)readInt(var0, var1) << 32) + ((long)readInt(var0, var1 + 4) & 4294967295L); + } + + public static long readLongLE(byte[] var0, int var1) { + return ((long)readIntLE(var0, var1) & 4294967295L) + ((long)readIntLE(var0, var1 + 4) << 32); + } + + public static double readDouble(byte[] var0, int var1) { + return Double.longBitsToDouble(readLong(var0, var1)); + } + + public static double readDoubleLE(byte[] var0, int var1) { + return Double.longBitsToDouble(readLongLE(var0, var1)); + } + + public static byte[] uuidToBytes(long var0, long var2) { + byte[] var4 = new byte[16]; + + for(int var5 = 0; var5 < 8; ++var5) { + var4[var5] = (byte)((int)(var0 >> 8 * (7 - var5) & 255L)); + var4[8 + var5] = (byte)((int)(var2 >> 8 * (7 - var5) & 255L)); + } + + return var4; + } + + public static byte[] uuidToBytes(UUID var0) { + return uuidToBytes(var0.getMostSignificantBits(), var0.getLeastSignificantBits()); + } + + public static void writeInt(byte[] var0, int var1, int var2) { + var0[var1++] = (byte)(var2 >> 24); + var0[var1++] = (byte)(var2 >> 16); + var0[var1++] = (byte)(var2 >> 8); + var0[var1] = (byte)var2; + } + + public static void writeIntLE(byte[] var0, int var1, int var2) { + var0[var1++] = (byte)var2; + var0[var1++] = (byte)(var2 >> 8); + var0[var1++] = (byte)(var2 >> 16); + var0[var1] = (byte)(var2 >> 24); + } + + public static void writeLong(byte[] var0, int var1, long var2) { + writeInt(var0, var1, (int)(var2 >> 32)); + writeInt(var0, var1 + 4, (int)var2); + } + + public static void writeLongLE(byte[] var0, int var1, long var2) { + writeIntLE(var0, var1, (int)var2); + writeIntLE(var0, var1 + 4, (int)(var2 >> 32)); + } + + public static void writeDouble(byte[] var0, int var1, double var2) { + writeLong(var0, var1, Double.doubleToRawLongBits(var2)); + } + + public static void writeDoubleLE(byte[] var0, int var1, double var2) { + writeLongLE(var0, var1, Double.doubleToRawLongBits(var2)); + } + + private Bits() { + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/IOUtils.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/IOUtils.java new file mode 100644 index 000000000..876fa8fa7 --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/IOUtils.java @@ -0,0 +1,273 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.util; + +import org.dizitart.no2.mvstore.compat.v1.mvstore.SysProperties; +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FileUtils; +import org.h2.jdbc.JdbcException; + +import java.io.*; +import java.nio.charset.StandardCharsets; + +public class IOUtils { + private IOUtils() { + } + + public static void closeSilently(AutoCloseable var0) { + if (var0 != null) { + try { + trace("closeSilently", (String)null, var0); + var0.close(); + } catch (Exception var2) { + } + } + + } + + public static void skipFully(InputStream var0, long var1) throws IOException { + try { + while(var1 > 0L) { + long var3 = var0.skip(var1); + if (var3 <= 0L) { + throw new EOFException(); + } + + var1 -= var3; + } + + } catch (Exception var5) { + throw convertToIOException(var5); + } + } + + public static void skipFully(Reader var0, long var1) throws IOException { + try { + while(var1 > 0L) { + long var3 = var0.skip(var1); + if (var3 <= 0L) { + throw new EOFException(); + } + + var1 -= var3; + } + + } catch (Exception var5) { + throw convertToIOException(var5); + } + } + + public static long copyAndClose(InputStream var0, OutputStream var1) throws IOException { + long var4; + try { + long var2 = copyAndCloseInput(var0, var1); + var1.close(); + var4 = var2; + } catch (Exception var9) { + throw convertToIOException(var9); + } finally { + closeSilently(var1); + } + + return var4; + } + + public static long copyAndCloseInput(InputStream var0, OutputStream var1) throws IOException { + long var2; + try { + var2 = copy(var0, var1); + } catch (Exception var7) { + throw convertToIOException(var7); + } finally { + closeSilently(var0); + } + + return var2; + } + + public static long copy(InputStream var0, OutputStream var1) throws IOException { + return copy(var0, var1, Long.MAX_VALUE); + } + + public static long copy(InputStream var0, OutputStream var1, long var2) throws IOException { + try { + long var4 = 0L; + int var6 = (int)Math.min(var2, 4096L); + + for(byte[] var7 = new byte[var6]; var2 > 0L; var6 = (int)Math.min(var2, 4096L)) { + var6 = var0.read(var7, 0, var6); + if (var6 < 0) { + break; + } + + if (var1 != null) { + var1.write(var7, 0, var6); + } + + var4 += (long)var6; + var2 -= (long)var6; + } + + return var4; + } catch (Exception var8) { + throw convertToIOException(var8); + } + } + + public static long copyAndCloseInput(Reader var0, Writer var1, long var2) throws IOException { + try { + long var4 = 0L; + int var6 = (int)Math.min(var2, 4096L); + char[] var7 = new char[var6]; + + while(true) { + if (var2 > 0L) { + var6 = var0.read(var7, 0, var6); + if (var6 >= 0) { + if (var1 != null) { + var1.write(var7, 0, var6); + } + + var2 -= (long)var6; + var6 = (int)Math.min(var2, 4096L); + var4 += (long)var6; + continue; + } + } + + long var8 = var4; + return var8; + } + } catch (Exception var13) { + throw convertToIOException(var13); + } finally { + var0.close(); + } + } + + public static byte[] readBytesAndClose(InputStream var0, int var1) throws IOException { + byte[] var4; + try { + if (var1 <= 0) { + var1 = Integer.MAX_VALUE; + } + + int var2 = Math.min(4096, var1); + ByteArrayOutputStream var3 = new ByteArrayOutputStream(var2); + copy(var0, var3, (long)var1); + var4 = var3.toByteArray(); + } catch (Exception var8) { + throw convertToIOException(var8); + } finally { + var0.close(); + } + + return var4; + } + + public static String readStringAndClose(Reader var0, int var1) throws IOException { + String var4; + try { + if (var1 <= 0) { + var1 = Integer.MAX_VALUE; + } + + int var2 = Math.min(4096, var1); + StringWriter var3 = new StringWriter(var2); + copyAndCloseInput(var0, var3, (long)var1); + var4 = var3.toString(); + } finally { + var0.close(); + } + + return var4; + } + + public static int readFully(InputStream var0, byte[] var1, int var2) throws IOException { + try { + int var3 = 0; + + int var5; + for(int var4 = Math.min(var2, var1.length); var4 > 0; var4 -= var5) { + var5 = var0.read(var1, var3, var4); + if (var5 < 0) { + break; + } + + var3 += var5; + } + + return var3; + } catch (Exception var6) { + throw convertToIOException(var6); + } + } + + public static int readFully(Reader var0, char[] var1, int var2) throws IOException { + try { + int var3 = 0; + + int var5; + for(int var4 = Math.min(var2, var1.length); var4 > 0; var4 -= var5) { + var5 = var0.read(var1, var3, var4); + if (var5 < 0) { + break; + } + + var3 += var5; + } + + return var3; + } catch (Exception var6) { + throw convertToIOException(var6); + } + } + + public static Reader getBufferedReader(InputStream var0) { + return var0 == null ? null : new BufferedReader(new InputStreamReader(var0, StandardCharsets.UTF_8)); + } + + public static Reader getReader(InputStream var0) { + return var0 == null ? null : new BufferedReader(new InputStreamReader(var0, StandardCharsets.UTF_8)); + } + + public static Writer getBufferedWriter(OutputStream var0) { + return var0 == null ? null : new BufferedWriter(new OutputStreamWriter(var0, StandardCharsets.UTF_8)); + } + + public static Reader getAsciiReader(InputStream var0) { + return var0 == null ? null : new InputStreamReader(var0, StandardCharsets.US_ASCII); + } + + public static void trace(String var0, String var1, Object var2) { + if (SysProperties.TRACE_IO) { + System.out.println("IOUtils." + var0 + ' ' + var1 + ' ' + var2); + } + + } + + public static InputStream getInputStreamFromString(String var0) { + return var0 == null ? null : new ByteArrayInputStream(var0.getBytes(StandardCharsets.UTF_8)); + } + + public static void copyFiles(String var0, String var1) throws IOException { + InputStream var2 = FileUtils.newInputStream(var0); + OutputStream var3 = FileUtils.newOutputStream(var1, false); + copyAndClose(var2, var3); + } + + public static IOException convertToIOException(Throwable var0) { + if (var0 instanceof IOException) { + return (IOException)var0; + } else { + if (var0 instanceof JdbcException && var0.getCause() != null) { + var0 = var0.getCause(); + } + + return new IOException(var0.toString(), var0); + } + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/SortedProperties.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/SortedProperties.java new file mode 100644 index 000000000..5e59ac4ee --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/SortedProperties.java @@ -0,0 +1,164 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.util; + + +import org.dizitart.no2.mvstore.compat.v1.mvstore.fs.FileUtils; +import org.h2.util.Utils; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +public class SortedProperties extends Properties { + private static final long serialVersionUID = 1L; + + public SortedProperties() { + } + + public synchronized Enumeration keys() { + Vector var1 = new Vector(); + Iterator var2 = this.keySet().iterator(); + + while(var2.hasNext()) { + Object var3 = var2.next(); + var1.add(var3.toString()); + } + + Collections.sort(var1); + return (new Vector(var1)).elements(); + } + + public static boolean getBooleanProperty(Properties var0, String var1, boolean var2) { + try { + return Utils.parseBoolean(var0.getProperty(var1, (String)null), var2, true); + } catch (IllegalArgumentException var4) { + var4.printStackTrace(); + return var2; + } + } + + public static int getIntProperty(Properties var0, String var1, int var2) { + String var3 = var0.getProperty(var1, Integer.toString(var2)); + + try { + return Integer.decode(var3); + } catch (Exception var5) { + var5.printStackTrace(); + return var2; + } + } + + public static String getStringProperty(Properties var0, String var1, String var2) { + return var0.getProperty(var1, var2); + } + + public static synchronized SortedProperties loadProperties(String var0) throws IOException { + SortedProperties var1 = new SortedProperties(); + if (FileUtils.exists(var0)) { + InputStream var2 = FileUtils.newInputStream(var0); + Throwable var3 = null; + + try { + var1.load(var2); + } catch (Throwable var12) { + var3 = var12; + throw var12; + } finally { + if (var2 != null) { + if (var3 != null) { + try { + var2.close(); + } catch (Throwable var11) { + var3.addSuppressed(var11); + } + } else { + var2.close(); + } + } + + } + } + + return var1; + } + + public synchronized void store(String var1) throws IOException { + ByteArrayOutputStream var2 = new ByteArrayOutputStream(); + this.store(var2, (String)null); + ByteArrayInputStream var3 = new ByteArrayInputStream(var2.toByteArray()); + InputStreamReader var4 = new InputStreamReader(var3, StandardCharsets.ISO_8859_1); + LineNumberReader var5 = new LineNumberReader(var4); + + OutputStreamWriter var6; + try { + var6 = new OutputStreamWriter(FileUtils.newOutputStream(var1, false)); + } catch (Exception var18) { + throw new IOException(var18.toString(), var18); + } + + PrintWriter var7 = new PrintWriter(new BufferedWriter(var6)); + Throwable var8 = null; + + try { + while(true) { + String var9 = var5.readLine(); + if (var9 == null) { + return; + } + + if (!var9.startsWith("#")) { + var7.print(var9 + "\n"); + } + } + } catch (Throwable var19) { + var8 = var19; + throw var19; + } finally { + if (var7 != null) { + if (var8 != null) { + try { + var7.close(); + } catch (Throwable var17) { + var8.addSuppressed(var17); + } + } else { + var7.close(); + } + } + + } + } + + public synchronized String toLines() { + StringBuilder var1 = new StringBuilder(); + Iterator var2 = (new TreeMap(this)).entrySet().iterator(); + + while(var2.hasNext()) { + Map.Entry var3 = (Map.Entry)var2.next(); + var1.append(var3.getKey()).append('=').append(var3.getValue()).append('\n'); + } + + return var1.toString(); + } + + public static SortedProperties fromLines(String var0) { + SortedProperties var1 = new SortedProperties(); + String[] var2 = StringUtils.arraySplit(var0, '\n', true); + int var3 = var2.length; + + for(int var4 = 0; var4 < var3; ++var4) { + String var5 = var2[var4]; + int var6 = var5.indexOf(61); + if (var6 > 0) { + var1.put(var5.substring(0, var6), var5.substring(var6 + 1)); + } + } + + return var1; + } +} diff --git a/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/StringUtils.java b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/StringUtils.java new file mode 100644 index 000000000..5094ac40c --- /dev/null +++ b/nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/compat/v1/mvstore/util/StringUtils.java @@ -0,0 +1,912 @@ +/* + * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0, + * and the EPL 1.0 (https://h2database.com/html/license.html). + * Initial Developer: H2 Group + */ + +package org.dizitart.no2.mvstore.compat.v1.mvstore.util; + +import org.h2.engine.SysProperties; +import org.h2.message.DbException; +import org.h2.util.Utils; + +import java.io.ByteArrayOutputStream; +import java.lang.ref.SoftReference; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +public class StringUtils { + private static SoftReference softCache; + private static long softCacheCreatedNs; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + private static final int[] HEX_DECODE = new int[103]; + private static final int TO_UPPER_CACHE_LENGTH = 2048; + private static final int TO_UPPER_CACHE_MAX_ENTRY_LENGTH = 64; + private static final String[][] TO_UPPER_CACHE = new String[2048][]; + + private StringUtils() { + } + + private static String[] getCache() { + String[] var0; + if (softCache != null) { + var0 = (String[])softCache.get(); + if (var0 != null) { + return var0; + } + } + + long var1 = System.nanoTime(); + if (softCacheCreatedNs != 0L && var1 - softCacheCreatedNs < TimeUnit.SECONDS.toNanos(5L)) { + return null; + } else { + String[] var3; + try { + var0 = new String[SysProperties.OBJECT_CACHE_SIZE]; + softCache = new SoftReference(var0); + var3 = var0; + } finally { + softCacheCreatedNs = System.nanoTime(); + } + + return var3; + } + } + + public static String toUpperEnglish(String var0) { + if (var0.length() > 64) { + return var0.toUpperCase(Locale.ENGLISH); + } else { + int var1 = var0.hashCode() & 2047; + String[] var2 = TO_UPPER_CACHE[var1]; + if (var2 != null && var2[0].equals(var0)) { + return var2[1]; + } else { + String var3 = var0.toUpperCase(Locale.ENGLISH); + var2 = new String[]{var0, var3}; + TO_UPPER_CACHE[var1] = var2; + return var3; + } + } + } + + public static String toLowerEnglish(String var0) { + return var0.toLowerCase(Locale.ENGLISH); + } + + public static String quoteStringSQL(String var0) { + return var0 == null ? "NULL" : quoteStringSQL(new StringBuilder(var0.length() + 2), var0).toString(); + } + + public static StringBuilder quoteStringSQL(StringBuilder var0, String var1) { + if (var1 == null) { + return var0.append("NULL"); + } else { + int var2 = var0.length(); + int var3 = var1.length(); + var0.append('\''); + + for(int var4 = 0; var4 < var3; ++var4) { + char var5 = var1.charAt(var4); + if (var5 == '\'') { + var0.append(var5); + } else if (var5 < ' ' || var5 > 127) { + var0.setLength(var2); + var0.append("STRINGDECODE('"); + javaEncode(var1, var0, true); + return var0.append("')"); + } + + var0.append(var5); + } + + return var0.append('\''); + } + } + + public static String javaEncode(String var0) { + StringBuilder var1 = new StringBuilder(var0.length()); + javaEncode(var0, var1, false); + return var1.toString(); + } + + public static void javaEncode(String var0, StringBuilder var1, boolean var2) { + int var3 = var0.length(); + + for(int var4 = 0; var4 < var3; ++var4) { + char var5 = var0.charAt(var4); + switch (var5) { + case '\t': + var1.append("\\t"); + break; + case '\n': + var1.append("\\n"); + break; + case '\f': + var1.append("\\f"); + break; + case '\r': + var1.append("\\r"); + break; + case '"': + var1.append("\\\""); + break; + case '\'': + if (var2) { + var1.append('\''); + } + + var1.append('\''); + break; + case '\\': + var1.append("\\\\"); + break; + default: + if (var5 >= ' ' && var5 < 128) { + var1.append(var5); + } else { + var1.append("\\u").append(HEX[var5 >>> 12]).append(HEX[var5 >>> 8 & 15]).append(HEX[var5 >>> 4 & 15]).append(HEX[var5 & 15]); + } + } + } + + } + + public static String addAsterisk(String var0, int var1) { + if (var0 != null) { + int var2 = var0.length(); + var1 = Math.min(var1, var2); + var0 = (new StringBuilder(var2 + 3)).append(var0, 0, var1).append("[*]").append(var0, var1, var2).toString(); + } + + return var0; + } + + private static DbException getFormatException(String var0, int var1) { + return DbException.get(90095, addAsterisk(var0, var1)); + } + + public static String javaDecode(String var0) { + int var1 = var0.length(); + StringBuilder var2 = new StringBuilder(var1); + + for(int var3 = 0; var3 < var1; ++var3) { + char var4 = var0.charAt(var3); + if (var4 == '\\') { + if (var3 + 1 >= var0.length()) { + throw getFormatException(var0, var3); + } + + ++var3; + var4 = var0.charAt(var3); + switch (var4) { + case '"': + var2.append('"'); + break; + case '#': + var2.append('#'); + break; + case ':': + var2.append(':'); + break; + case '=': + var2.append('='); + break; + case '\\': + var2.append('\\'); + break; + case 'b': + var2.append('\b'); + break; + case 'f': + var2.append('\f'); + break; + case 'n': + var2.append('\n'); + break; + case 'r': + var2.append('\r'); + break; + case 't': + var2.append('\t'); + break; + case 'u': + try { + var4 = (char)Integer.parseInt(var0.substring(var3 + 1, var3 + 5), 16); + } catch (NumberFormatException var7) { + throw getFormatException(var0, var3); + } + + var3 += 4; + var2.append(var4); + break; + default: + if (var4 < '0' || var4 > '9') { + throw getFormatException(var0, var3); + } + + try { + var4 = (char)Integer.parseInt(var0.substring(var3, var3 + 3), 8); + } catch (NumberFormatException var6) { + throw getFormatException(var0, var3); + } + + var3 += 2; + var2.append(var4); + } + } else { + var2.append(var4); + } + } + + return var2.toString(); + } + + public static String quoteJavaString(String var0) { + if (var0 == null) { + return "null"; + } else { + StringBuilder var1 = (new StringBuilder(var0.length() + 2)).append('"'); + javaEncode(var0, var1, false); + return var1.append('"').toString(); + } + } + + public static String quoteJavaStringArray(String[] var0) { + if (var0 == null) { + return "null"; + } else { + StringBuilder var1 = new StringBuilder("new String[]{"); + + for(int var2 = 0; var2 < var0.length; ++var2) { + if (var2 > 0) { + var1.append(", "); + } + + var1.append(quoteJavaString(var0[var2])); + } + + return var1.append('}').toString(); + } + } + + public static String quoteJavaIntArray(int[] var0) { + if (var0 == null) { + return "null"; + } else { + StringBuilder var1 = new StringBuilder("new int[]{"); + + for(int var2 = 0; var2 < var0.length; ++var2) { + if (var2 > 0) { + var1.append(", "); + } + + var1.append(var0[var2]); + } + + return var1.append('}').toString(); + } + } + + public static String unEnclose(String var0) { + return var0.startsWith("(") && var0.endsWith(")") ? var0.substring(1, var0.length() - 1) : var0; + } + + public static String urlEncode(String var0) { + try { + return URLEncoder.encode(var0, "UTF-8"); + } catch (Exception var2) { + throw DbException.convert(var2); + } + } + + public static String urlDecode(String var0) { + int var1 = var0.length(); + byte[] var2 = new byte[var1]; + int var3 = 0; + + for(int var4 = 0; var4 < var1; ++var4) { + char var5 = var0.charAt(var4); + if (var5 == '+') { + var2[var3++] = 32; + } else if (var5 == '%') { + var2[var3++] = (byte)Integer.parseInt(var0.substring(var4 + 1, var4 + 3), 16); + var4 += 2; + } else { + if (var5 > 127 || var5 < ' ') { + throw new IllegalArgumentException("Unexpected char " + var5 + " decoding " + var0); + } + + var2[var3++] = (byte)var5; + } + } + + return new String(var2, 0, var3, StandardCharsets.UTF_8); + } + + public static String[] arraySplit(String var0, char var1, boolean var2) { + if (var0 == null) { + return null; + } else { + int var3 = var0.length(); + if (var3 == 0) { + return new String[0]; + } else { + ArrayList var4 = Utils.newSmallArrayList(); + StringBuilder var5 = new StringBuilder(var3); + + for(int var6 = 0; var6 < var3; ++var6) { + char var7 = var0.charAt(var6); + if (var7 == var1) { + String var8 = var5.toString(); + var4.add(var2 ? var8.trim() : var8); + var5.setLength(0); + } else if (var7 == '\\' && var6 < var3 - 1) { + ++var6; + var5.append(var0.charAt(var6)); + } else { + var5.append(var7); + } + } + + String var9 = var5.toString(); + var4.add(var2 ? var9.trim() : var9); + return (String[])var4.toArray(new String[0]); + } + } + } + + public static String arrayCombine(String[] var0, char var1) { + StringBuilder var2 = new StringBuilder(); + + for(int var3 = 0; var3 < var0.length; ++var3) { + if (var3 > 0) { + var2.append(var1); + } + + String var4 = var0[var3]; + if (var4 != null) { + int var5 = 0; + + for(int var6 = var4.length(); var5 < var6; ++var5) { + char var7 = var4.charAt(var5); + if (var7 == '\\' || var7 == var1) { + var2.append('\\'); + } + + var2.append(var7); + } + } + } + + return var2.toString(); + } + + public static StringBuilder join(StringBuilder var0, ArrayList var1, String var2) { + int var3 = 0; + + for(int var4 = var1.size(); var3 < var4; ++var3) { + if (var3 > 0) { + var0.append(var2); + } + + var0.append((String)var1.get(var3)); + } + + return var0; + } + + public static String xmlAttr(String var0, String var1) { + return " " + var0 + "=\"" + xmlText(var1) + "\""; + } + + public static String xmlNode(String var0, String var1, String var2) { + return xmlNode(var0, var1, var2, true); + } + + public static String xmlNode(String var0, String var1, String var2, boolean var3) { + StringBuilder var4 = new StringBuilder(); + var4.append('<').append(var0); + if (var1 != null) { + var4.append(var1); + } + + if (var2 == null) { + var4.append("/>\n"); + return var4.toString(); + } else { + var4.append('>'); + if (var3 && var2.indexOf(10) >= 0) { + var4.append('\n'); + indent(var4, var2, 4, true); + } else { + var4.append(var2); + } + + var4.append("\n"); + return var4.toString(); + } + } + + public static StringBuilder indent(StringBuilder var0, String var1, int var2, boolean var3) { + int var4 = 0; + + int var6; + for(int var5 = var1.length(); var4 < var5; var4 = var6) { + for(var6 = 0; var6 < var2; ++var6) { + var0.append(' '); + } + + var6 = var1.indexOf(10, var4); + var6 = var6 < 0 ? var5 : var6 + 1; + var0.append(var1, var4, var6); + } + + if (var3 && !var1.endsWith("\n")) { + var0.append('\n'); + } + + return var0; + } + + public static String xmlComment(String var0) { + int var1 = 0; + + while(true) { + var1 = var0.indexOf("--", var1); + if (var1 < 0) { + if (var0.indexOf(10) >= 0) { + StringBuilder var2 = (new StringBuilder(var0.length() + 18)).append("\n").toString(); + } else { + return "\n"; + } + } + + var0 = var0.substring(0, var1 + 1) + " " + var0.substring(var1 + 1); + } + } + + public static String xmlCData(String var0) { + if (var0.contains("]]>")) { + return xmlText(var0); + } else { + boolean var1 = var0.endsWith("\n"); + var0 = ""; + return var1 ? var0 + "\n" : var0; + } + } + + public static String xmlStartDoc() { + return "\n"; + } + + public static String xmlText(String var0) { + return xmlText(var0, false); + } + + public static String xmlText(String var0, boolean var1) { + int var2 = var0.length(); + StringBuilder var3 = new StringBuilder(var2); + + for(int var4 = 0; var4 < var2; ++var4) { + char var5 = var0.charAt(var4); + switch (var5) { + case '\t': + var3.append(var5); + break; + case '\n': + case '\r': + if (var1) { + var3.append("&#x").append(Integer.toHexString(var5)).append(';'); + } else { + var3.append(var5); + } + break; + case '"': + var3.append("""); + break; + case '&': + var3.append("&"); + break; + case '\'': + var3.append("'"); + break; + case '<': + var3.append("<"); + break; + case '>': + var3.append(">"); + break; + default: + if (var5 >= ' ' && var5 <= 127) { + var3.append(var5); + } else { + var3.append("&#x").append(Integer.toHexString(var5)).append(';'); + } + } + } + + return var3.toString(); + } + + public static String replaceAll(String var0, String var1, String var2) { + int var3 = var0.indexOf(var1); + if (var3 >= 0 && !var1.isEmpty()) { + StringBuilder var4 = new StringBuilder(var0.length() - var1.length() + var2.length()); + int var5 = 0; + + do { + var4.append(var0, var5, var3).append(var2); + var5 = var3 + var1.length(); + var3 = var0.indexOf(var1, var5); + } while(var3 >= 0); + + var4.append(var0, var5, var0.length()); + return var4.toString(); + } else { + return var0; + } + } + + public static String quoteIdentifier(String var0) { + return quoteIdentifier(new StringBuilder(var0.length() + 2), var0).toString(); + } + + public static StringBuilder quoteIdentifier(StringBuilder var0, String var1) { + var0.append('"'); + int var2 = 0; + + for(int var3 = var1.length(); var2 < var3; ++var2) { + char var4 = var1.charAt(var2); + if (var4 == '"') { + var0.append(var4); + } + + var0.append(var4); + } + + return var0.append('"'); + } + + public static boolean isNullOrEmpty(String var0) { + return var0 == null || var0.isEmpty(); + } + + public static String quoteRemarkSQL(String var0) { + var0 = replaceAll(var0, "*/", "++/"); + return replaceAll(var0, "/*", "/++"); + } + + public static String pad(String var0, int var1, String var2, boolean var3) { + if (var1 < 0) { + var1 = 0; + } + + if (var1 < var0.length()) { + return var0.substring(0, var1); + } else if (var1 == var0.length()) { + return var0; + } else { + char var4; + if (var2 != null && !var2.isEmpty()) { + var4 = var2.charAt(0); + } else { + var4 = ' '; + } + + StringBuilder var5 = new StringBuilder(var1); + var1 -= var0.length(); + if (var3) { + var5.append(var0); + } + + for(int var6 = 0; var6 < var1; ++var6) { + var5.append(var4); + } + + if (!var3) { + var5.append(var0); + } + + return var5.toString(); + } + } + + public static char[] cloneCharArray(char[] var0) { + if (var0 == null) { + return null; + } else { + int var1 = var0.length; + return var1 == 0 ? var0 : Arrays.copyOf(var0, var1); + } + } + + public static String trim(String var0, boolean var1, boolean var2, String var3) { + char var4 = var3 != null && !var3.isEmpty() ? var3.charAt(0) : 32; + int var5 = 0; + int var6 = var0.length(); + if (var1) { + while(var5 < var6 && var0.charAt(var5) == var4) { + ++var5; + } + } + + if (var2) { + while(var6 > var5 && var0.charAt(var6 - 1) == var4) { + --var6; + } + } + + return var0.substring(var5, var6); + } + + public static String trimSubstring(String var0, int var1) { + return trimSubstring(var0, var1, var0.length()); + } + + public static String trimSubstring(String var0, int var1, int var2) { + while(var1 < var2 && var0.charAt(var1) <= ' ') { + ++var1; + } + + while(var1 < var2 && var0.charAt(var2 - 1) <= ' ') { + --var2; + } + + return var0.substring(var1, var2); + } + + public static StringBuilder trimSubstring(StringBuilder var0, String var1, int var2, int var3) { + while(var2 < var3 && var1.charAt(var2) <= ' ') { + ++var2; + } + + while(var2 < var3 && var1.charAt(var3 - 1) <= ' ') { + --var3; + } + + return var0.append(var1, var2, var3); + } + + public static String cache(String var0) { + if (!SysProperties.OBJECT_CACHE) { + return var0; + } else if (var0 == null) { + return var0; + } else if (var0.isEmpty()) { + return ""; + } else { + String[] var1 = getCache(); + if (var1 != null) { + int var2 = var0.hashCode(); + int var3 = var2 & SysProperties.OBJECT_CACHE_SIZE - 1; + String var4 = var1[var3]; + if (var0.equals(var4)) { + return var4; + } + + var1[var3] = var0; + } + + return var0; + } + } + + public static void clearCache() { + softCache = null; + } + + public static int parseUInt31(String var0, int var1, int var2) { + if (var2 <= var0.length() && var1 >= 0 && var1 <= var2) { + if (var1 == var2) { + throw new NumberFormatException(""); + } else { + int var3 = 0; + + for(int var4 = var1; var4 < var2; ++var4) { + char var5 = var0.charAt(var4); + if (var5 < '0' || var5 > '9' || var3 > 214748364) { + throw new NumberFormatException(var0.substring(var1, var2)); + } + + var3 = var3 * 10 + var5 - 48; + if (var3 < 0) { + throw new NumberFormatException(var0.substring(var1, var2)); + } + } + + return var3; + } + } else { + throw new IndexOutOfBoundsException(); + } + } + + public static byte[] convertHexToBytes(String var0) { + int var1 = var0.length(); + if (var1 % 2 != 0) { + throw DbException.get(90003, var0); + } else { + var1 /= 2; + byte[] var2 = new byte[var1]; + int var3 = 0; + int[] var4 = HEX_DECODE; + + try { + for(int var5 = 0; var5 < var1; ++var5) { + int var6 = var4[var0.charAt(var5 + var5)] << 4 | var4[var0.charAt(var5 + var5 + 1)]; + var3 |= var6; + var2[var5] = (byte)var6; + } + } catch (ArrayIndexOutOfBoundsException var7) { + throw DbException.get(90004, var0); + } + + if ((var3 & -256) != 0) { + throw DbException.get(90004, var0); + } else { + return var2; + } + } + } + + public static ByteArrayOutputStream convertHexWithSpacesToBytes(ByteArrayOutputStream var0, String var1) { + int var2 = var1.length(); + if (var0 == null) { + var0 = new ByteArrayOutputStream(var2 / 2); + } + + int var3 = 0; + int[] var4 = HEX_DECODE; + + try { + int var5 = 0; + + while(var5 < var2) { + char var6 = var1.charAt(var5++); + if (var6 != ' ') { + char var7; + do { + if (var5 >= var2) { + if (((var3 | var4[var6]) & -256) != 0) { + throw DbException.get(90004, var1); + } + + throw DbException.get(90003, var1); + } + + var7 = var1.charAt(var5++); + } while(var7 == ' '); + + int var8 = var4[var6] << 4 | var4[var7]; + var3 |= var8; + var0.write(var8); + } + } + } catch (ArrayIndexOutOfBoundsException var9) { + throw DbException.get(90004, var1); + } + + if ((var3 & -256) != 0) { + throw DbException.get(90004, var1); + } else { + return var0; + } + } + + public static String convertBytesToHex(byte[] var0) { + return convertBytesToHex(var0, var0.length); + } + + public static String convertBytesToHex(byte[] var0, int var1) { + char[] var2 = new char[var1 + var1]; + char[] var3 = HEX; + + for(int var4 = 0; var4 < var1; ++var4) { + int var5 = var0[var4] & 255; + var2[var4 + var4] = var3[var5 >> 4]; + var2[var4 + var4 + 1] = var3[var5 & 15]; + } + + return new String(var2); + } + + public static StringBuilder convertBytesToHex(StringBuilder var0, byte[] var1) { + return convertBytesToHex(var0, var1, var1.length); + } + + public static StringBuilder convertBytesToHex(StringBuilder var0, byte[] var1, int var2) { + char[] var3 = HEX; + + for(int var4 = 0; var4 < var2; ++var4) { + int var5 = var1[var4] & 255; + var0.append(var3[var5 >>> 4]).append(var3[var5 & 15]); + } + + return var0; + } + + public static StringBuilder appendHex(StringBuilder var0, long var1, int var3) { + char[] var4 = HEX; + int var5 = var3 * 8; + + while(var5 > 0) { + var5 -= 4; + StringBuilder var10000 = var0.append(var4[(int)(var1 >> var5) & 15]); + var5 -= 4; + var10000.append(var4[(int)(var1 >> var5) & 15]); + } + + return var0; + } + + public static boolean isNumber(String var0) { + int var1 = var0.length(); + if (var1 == 0) { + return false; + } else { + for(int var2 = 0; var2 < var1; ++var2) { + if (!Character.isDigit(var0.charAt(var2))) { + return false; + } + } + + return true; + } + } + + public static boolean isWhitespaceOrEmpty(String var0) { + int var1 = 0; + + for(int var2 = var0.length(); var1 < var2; ++var1) { + if (var0.charAt(var1) > ' ') { + return false; + } + } + + return true; + } + + public static void appendZeroPadded(StringBuilder var0, int var1, long var2) { + if (var1 == 2) { + if (var2 < 10L) { + var0.append('0'); + } + + var0.append(var2); + } else { + String var4 = Long.toString(var2); + + for(var1 -= var4.length(); var1 > 0; --var1) { + var0.append('0'); + } + + var0.append(var4); + } + + } + + public static String escapeMetaDataPattern(String var0) { + return var0 != null && !var0.isEmpty() ? replaceAll(var0, "\\", "\\\\") : var0; + } + + static { + int var0; + for(var0 = 0; var0 < HEX_DECODE.length; ++var0) { + HEX_DECODE[var0] = -1; + } + + for(var0 = 0; var0 <= 9; HEX_DECODE[var0 + 48] = var0++) { + } + + for(var0 = 0; var0 <= 5; ++var0) { + HEX_DECODE[var0 + 97] = HEX_DECODE[var0 + 65] = var0 + 10; + } + + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/NitriteTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/NitriteTest.java index d49a599cf..a38e8fdba 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/NitriteTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/NitriteTest.java @@ -27,14 +27,16 @@ import org.dizitart.no2.common.SortOrder; import org.dizitart.no2.common.WriteResult; import org.dizitart.no2.common.concurrent.ThreadPoolManager; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.ValidationException; import org.dizitart.no2.index.IndexOptions; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.TestUtil; +import org.dizitart.no2.integration.repository.data.EmptyClass; import org.dizitart.no2.mvstore.MVStoreModule; import org.dizitart.no2.repository.ObjectRepository; import org.dizitart.no2.repository.annotations.Id; @@ -84,6 +86,11 @@ public class NitriteTest { public void setUp() throws ParseException { db = TestUtil.createDb(fileName, "test-user", "test-password"); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new CompatChild.Converter()); + documentMapper.registerEntityConverter(new Receipt.Converter()); + documentMapper.registerEntityConverter(new EmptyClass.Converter()); + simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH); Document doc1 = createDocument("firstName", "fn1") @@ -132,9 +139,14 @@ public void testListCollectionNames() { assertEquals(collectionNames.size(), 1); } - @Test + @Test(expected = ValidationException.class) public void testListRepositories() { db.getRepository(getClass()); + } + + @Test + public void testListRepositories2() { + db.getRepository(Receipt.class); Set repositories = db.listRepositories(); assertEquals(repositories.size(), 1); } @@ -145,10 +157,15 @@ public void testHasCollection() { assertFalse(db.hasCollection("lucene" + INTERNAL_NAME_SEPARATOR + "test")); } - @Test + @Test(expected = ValidationException.class) public void testHasRepository() { db.getRepository(getClass()); - assertTrue(db.hasRepository(getClass())); + } + + @Test + public void testHasRepository2() { + db.getRepository(Receipt.class); + assertTrue(db.hasRepository(Receipt.class)); assertFalse(db.hasRepository(String.class)); } @@ -240,20 +257,30 @@ public void testGetCollection() { assertEquals(collection.getName(), "test-collection"); } - @Test + @Test(expected = ValidationException.class) public void testGetRepository() { - ObjectRepository repository = db.getRepository(NitriteTest.class); - assertNotNull(repository); - assertEquals(repository.getType(), NitriteTest.class); + ObjectRepository repository = db.getRepository(EmptyClass.class); } @Test + public void testGetRepository2() { + ObjectRepository repository = db.getRepository(Receipt.class); + assertNotNull(repository); + assertEquals(repository.getType(), Receipt.class); + } + + @Test(expected = ValidationException.class) public void testGetRepositoryWithKey() { - ObjectRepository repository = db.getRepository(NitriteTest.class, "key"); + ObjectRepository repository = db.getRepository(EmptyClass.class, "key"); + } + + @Test + public void testGetRepositoryWithKey2() { + ObjectRepository repository = db.getRepository(Receipt.class, "key"); assertNotNull(repository); - assertEquals(repository.getType(), NitriteTest.class); - assertFalse(db.hasRepository(NitriteTest.class)); - assertTrue(db.hasRepository(NitriteTest.class, "key")); + assertEquals(repository.getType(), Receipt.class); + assertFalse(db.hasRepository(Receipt.class)); + assertTrue(db.hasRepository(Receipt.class, "key")); } @Test @@ -269,18 +296,18 @@ public void testMultipleGetCollection() { @Test public void testMultipleGetRepository() { - ObjectRepository repository = db.getRepository(NitriteTest.class); + ObjectRepository repository = db.getRepository(Receipt.class); assertNotNull(repository); - assertEquals(repository.getType(), NitriteTest.class); + assertEquals(repository.getType(), Receipt.class); - ObjectRepository repository2 = db.getRepository(NitriteTest.class); + ObjectRepository repository2 = db.getRepository(Receipt.class); assertNotNull(repository2); - assertEquals(repository2.getType(), NitriteTest.class); + assertEquals(repository2.getType(), Receipt.class); } @Test(expected = ValidationException.class) public void testGetRepositoryInvalid() { - db.getRepository(null); + db.getRepository((Class) null); } @Test(expected = NitriteIOException.class) @@ -464,10 +491,12 @@ public void testReadCompatibility() throws IOException { String oldDbFile = System.getProperty("java.io.tmpdir") + File.separator + "old.db"; Nitrite db = TestUtil.createDb(oldDbFile, "test-user", "test-password"); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new Receipt.Converter()); NitriteCollection collection = db.getCollection("test"); - // text filter has be the first filter in and clause + // text filter has been the first filter in and clause List cursor = collection.find( and(where("second_key").text("fox"), where("first_key").eq(1))).toList(); assertEquals(cursor.size(), 1); @@ -565,20 +594,30 @@ public void run() { @Data @AllArgsConstructor @NoArgsConstructor - public static class CompatChild implements Mappable { + public static class CompatChild { private Long childId; private String lastName; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("childId", childId) - .put("lastName", lastName); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return CompatChild.class; + } - @Override - public void read(NitriteMapper mapper, Document document) { - childId = document.get("childId", Long.class); - lastName = document.get("lastName", String.class); + @Override + public Document toDocument(CompatChild entity, NitriteMapper nitriteMapper) { + return Document.createDocument("childId", entity.childId) + .put("lastName", entity.lastName); + } + + @Override + public CompatChild fromDocument(Document document, NitriteMapper nitriteMapper) { + CompatChild entity = new CompatChild(); + entity.childId = document.get("childId", Long.class); + entity.lastName = document.get("lastName", String.class); + return entity; + } } } @@ -586,35 +625,47 @@ public void read(NitriteMapper mapper, Document document) { @NoArgsConstructor @AllArgsConstructor @Indices({ - @Index(value = "synced", type = IndexType.NON_UNIQUE) + @Index(fields = "synced", type = IndexType.NON_UNIQUE) }) - public static class Receipt implements Mappable { + public static class Receipt { @Id private String clientRef; private Boolean synced; private Status status; private Long createdTimestamp = System.currentTimeMillis(); - @Override - public Document write(NitriteMapper mapper) { - return createDocument("status", status) - .put("clientRef", clientRef) - .put("synced", synced) - .put("createdTimestamp", createdTimestamp); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return Receipt.class; + } - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - Object status = document.get("status"); - if (status instanceof Status) { - this.status = (Status) status; - } else { - this.status = Status.valueOf(status.toString()); + @Override + public Document toDocument(Receipt entity, NitriteMapper nitriteMapper) { + return createDocument("status", entity.status) + .put("clientRef", entity.clientRef) + .put("synced", entity.synced) + .put("createdTimestamp", entity.createdTimestamp); + } + + @Override + public Receipt fromDocument(Document document, NitriteMapper nitriteMapper) { + Receipt receipt = new Receipt(); + if (document != null) { + Object status = document.get("status"); + if (status != null) { + if (status instanceof Receipt.Status) { + receipt.status = (Receipt.Status) status; + } else { + receipt.status = Receipt.Status.valueOf(status.toString()); + } + } + receipt.clientRef = document.get("clientRef", String.class); + receipt.synced = document.get("synced", Boolean.class); + receipt.createdTimestamp = document.get("createdTimestamp", Long.class); } - this.clientRef = document.get("clientRef", String.class); - this.synced = document.get("synced", Boolean.class); - this.createdTimestamp = document.get("createdTimestamp", Long.class); + return receipt; } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java index fc5457ee5..4565edd6c 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java @@ -26,8 +26,9 @@ import org.dizitart.no2.collection.NitriteId; import org.dizitart.no2.common.FieldValues; import org.dizitart.no2.common.Fields; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.NitriteSecurityException; @@ -193,6 +194,10 @@ public void testPopulateRepositories() { .loadModule(module) .openOrCreate(); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TestObject.Converter()); + documentMapper.registerEntityConverter(new TestObject2.Converter()); + NitriteCollection collection = db.getCollection("test"); collection.insert(createDocument("id1", "value")); @@ -368,24 +373,14 @@ public Target convert(Source source, Class type) { return null; } - @Override - public boolean isValueType(Class type) { - return false; - } - - @Override - public boolean isValue(Object object) { - return false; - } - @Override public void initialize(NitriteConfig nitriteConfig) { } } - @Index(value = "longValue") - private static class TestObject implements Mappable { + @Index(fields = "longValue") + private static class TestObject { private String stringValue; private Long longValue; @@ -397,23 +392,32 @@ public TestObject(String stringValue, Long longValue) { this.stringValue = stringValue; } - @Override - public Document write(NitriteMapper mapper) { - return createDocument("stringValue", stringValue) - .put("longValue", longValue); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - this.stringValue = document.get("stringValue", String.class); - this.longValue = document.get("longValue", Long.class); + @Override + public Class getEntityType() { + return TestObject.class; + } + + @Override + public Document toDocument(TestObject entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("stringValue", entity.stringValue) + .put("longValue", entity.longValue); + } + + @Override + public TestObject fromDocument(Document document, NitriteMapper nitriteMapper) { + TestObject entity = new TestObject(); + entity.stringValue = document.get("stringValue", String.class); + entity.longValue = document.get("longValue", Long.class); + return entity; } } } - @Index(value = "longValue") - private static class TestObject2 implements Mappable { + @Index(fields = "longValue") + private static class TestObject2 { private String stringValue; private Long longValue; @@ -425,17 +429,27 @@ public TestObject2(String stringValue, Long longValue) { this.stringValue = stringValue; } - @Override - public Document write(NitriteMapper mapper) { - return createDocument("stringValue", stringValue) - .put("longValue", longValue); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - this.stringValue = document.get("stringValue", String.class); - this.longValue = document.get("longValue", Long.class); + @Override + public Class getEntityType() { + return TestObject2.class; + } + + @Override + public Document toDocument(TestObject2 entity, NitriteMapper nitriteMapper) { + return createDocument("stringValue", entity.stringValue) + .put("longValue", entity.longValue); + } + + @Override + public TestObject2 fromDocument(Document document, NitriteMapper nitriteMapper) { + TestObject2 entity = new TestObject2(); + if (document != null) { + entity.stringValue = document.get("stringValue", String.class); + entity.longValue = document.get("longValue", Long.class); + } + return entity; } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java index 222038b62..d4bef6baf 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java @@ -55,7 +55,7 @@ public void testOpenSecuredWithoutCredential() { assertEquals(dbCollection.find().size(), 1); } - @Test(expected = NitriteException.class) + @Test public void testOpenUnsecuredWithCredential() { db = createDb(fileName); NitriteCollection dbCollection = db.getCollection("test"); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java index 19c4a830a..f556c9c8c 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java @@ -24,8 +24,9 @@ import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.DocumentCursor; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.filters.Filter; import org.dizitart.no2.index.IndexOptions; import org.dizitart.no2.index.IndexType; @@ -66,6 +67,11 @@ public class NitriteStressTest { @Before public void before() { db = createDb(fileName); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TestDto.Converter()); + documentMapper.registerEntityConverter(new PerfTest.Converter()); + documentMapper.registerEntityConverter(new PerfTestIndexed.Converter()); + collection = db.getCollection("test"); System.out.println(fileName); } @@ -84,6 +90,8 @@ public void cleanUp() { @Test public void stressTest() { ObjectRepository testRepository = db.getRepository(TestDto.class); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TestDto.Converter()); testRepository.createIndex(IndexOptions.indexOptions(IndexType.FULL_TEXT), "lastName"); testRepository.createIndex(IndexOptions.indexOptions(IndexType.NON_UNIQUE), "birthDate"); @@ -220,7 +228,7 @@ private List getItems(Class type) { } @Data - public static class TestDto implements Mappable { + public static class TestDto { @XmlElement( name = "StudentNumber", @@ -268,61 +276,109 @@ public static class TestDto implements Mappable { public TestDto() { } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("studentNumber", studentNumber) - .put("lastName", lastName) - .put("prefixes", prefixes) - .put("initials", initials) - .put("firstNames", firstNames) - .put("nickName", nickName) - .put("birthDate", birthDate); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return TestDto.class; + } + + @Override + public Document toDocument(TestDto entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("studentNumber", entity.studentNumber) + .put("lastName", entity.lastName) + .put("prefixes", entity.prefixes) + .put("initials", entity.initials) + .put("firstNames", entity.firstNames) + .put("nickName", entity.nickName) + .put("birthDate", entity.birthDate); + } - @Override - public void read(NitriteMapper mapper, Document document) { - studentNumber = document.get("studentNumber", String.class); - lastName = document.get("lastName", String.class); - prefixes = document.get("prefixes", String.class); - initials = document.get("initials", String.class); - firstNames = document.get("firstNames", String.class); - nickName = document.get("nickName", String.class); - birthDate = document.get("birthDate", String.class); + @Override + public TestDto fromDocument(Document document, NitriteMapper nitriteMapper) { + TestDto entity = new TestDto(); + entity.studentNumber = document.get("studentNumber", String.class); + entity.lastName = document.get("lastName", String.class); + entity.prefixes = document.get("prefixes", String.class); + entity.initials = document.get("initials", String.class); + entity.firstNames = document.get("firstNames", String.class); + entity.nickName = document.get("nickName", String.class); + entity.birthDate = document.get("birthDate", String.class); + return entity; + } } } @Data - public static class PerfTest implements Mappable { + public static class PerfTest { private String firstName; private String lastName; private Integer age; private String text; - @Override - public Document write(NitriteMapper mapper) { - Document document = Document.createDocument(); - document.put("firstName", firstName); - document.put("lastName", lastName); - document.put("age", age); - document.put("text", text); - return document; - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - this.firstName = (String) document.get("firstName"); - this.lastName = (String) document.get("lastName"); - this.age = (Integer) document.get("age"); - this.text = (String) document.get("text"); + @Override + public Class getEntityType() { + return PerfTest.class; + } + + @Override + public Document toDocument(PerfTest entity, NitriteMapper nitriteMapper) { + Document document = Document.createDocument(); + document.put("firstName", entity.firstName); + document.put("lastName", entity.lastName); + document.put("age", entity.age); + document.put("text", entity.text); + return document; + } + + @Override + public PerfTest fromDocument(Document document, NitriteMapper nitriteMapper) { + PerfTest entity = new PerfTest(); + entity.firstName = (String) document.get("firstName"); + entity.lastName = (String) document.get("lastName"); + entity.age = (Integer) document.get("age"); + entity.text = (String) document.get("text"); + return entity; + } } } @Indices({ - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "age", type = IndexType.NON_UNIQUE), - @Index(value = "text", type = IndexType.FULL_TEXT), + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "age", type = IndexType.NON_UNIQUE), + @Index(fields = "text", type = IndexType.FULL_TEXT), }) private static class PerfTestIndexed extends PerfTest { + + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return PerfTestIndexed.class; + } + + @Override + public Document toDocument(PerfTestIndexed entity, NitriteMapper nitriteMapper) { + Document document = Document.createDocument(); + document.put("firstName", entity.getFirstName()); + document.put("lastName", entity.getLastName()); + document.put("age", entity.getAge()); + document.put("text", entity.getText()); + return document; + } + + @Override + public PerfTestIndexed fromDocument(Document document, NitriteMapper nitriteMapper) { + PerfTestIndexed entity = new PerfTestIndexed(); + entity.setFirstName((String) document.get("firstName")); + entity.setLastName((String) document.get("lastName")); + entity.setAge((Integer) document.get("age")); + entity.setText((String) document.get("text")); + return entity; + } + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java index f5396fdfe..449b7ffe0 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java @@ -19,7 +19,9 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.repository.data.ClassA; +import org.dizitart.no2.integration.repository.data.ClassBConverter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -37,6 +39,11 @@ public class NitriteTest { @Before public void setUp() { db = createDb(fileName); + SimpleDocumentMapper nitriteMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + nitriteMapper.registerEntityConverter(new ClassA.ClassAConverter()); + nitriteMapper.registerEntityConverter(new ClassBConverter()); + + NitriteCollection collection = db.getCollection("test"); assertNotNull(collection); } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java index 6898e1169..200d7c873 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java @@ -104,7 +104,7 @@ public void setUp() { .put("lastName", "ln2") .put("birthDay", simpleDateFormat.parse("2010-06-12T16:02:48.440Z")) .put("data", new byte[]{3, 4, 3}) - .put("list", Arrays.asList("three", "four", "three")) + .put("list", Arrays.asList("three", "four", "five")) .put("body", "quick hello world from nitrite"); doc3 = createDocument("firstName", "fn3") .put("lastName", "ln2") diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java index 8fd30693e..f1c1d927e 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java @@ -19,6 +19,7 @@ import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.DocumentCursor; +import org.dizitart.no2.collection.FindOptions; import org.dizitart.no2.collection.FindPlan; import org.dizitart.no2.common.SortOrder; import org.dizitart.no2.common.tuples.Pair; @@ -82,7 +83,7 @@ public void testFindByOrFilterAndFilter() { where("firstName").eq("fn3"), where("lastName").eq("ln2") ) - ) + ), FindOptions.withDistinct() ); assertEquals(2, cursor.size()); @@ -199,6 +200,22 @@ public void testFindByOrFilter() throws ParseException { FindPlan findPlan = cursor.getFindPlan(); assertEquals(3, findPlan.getSubPlans().size()); + assertEquals(5, cursor.size()); + + // distinct + cursor = collection.find( + or( + or( + where("lastName").eq("ln2"), + where("firstName").notEq("fn1") + ), + where("birthDay").eq(simpleDateFormat.parse("2012-07-01T16:02:48.440Z")), + where("firstName").notEq("fn1") + ), FindOptions.withDistinct() + ); + + findPlan = cursor.getFindPlan(); + assertEquals(3, findPlan.getSubPlans().size()); assertEquals(3, cursor.size()); } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java index 8d446cf0f..71a97a42d 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java @@ -21,6 +21,7 @@ import org.dizitart.no2.collection.DocumentCursor; import org.dizitart.no2.common.SortOrder; import org.dizitart.no2.exceptions.FilterException; +import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.ValidationException; import org.junit.Test; @@ -61,7 +62,7 @@ public void testFindOptionsInvalidOffset() { assertEquals(collection.find(skipBy(10).limit(1)).size(), 0); } - @Test(expected = ValidationException.class) + @Test(expected = InvalidOperationException.class) public void testFindInvalidSort() { insert(); collection.find(orderBy("data", SortOrder.Descending)).toList(); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java index f5e5a2c04..7dedc072f 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java @@ -420,7 +420,7 @@ public void testFindWithIterableEqual() { new ArrayList() {{ add("three"); add("four"); - add("three"); + add("five"); }})); assertNotNull(ids); assertEquals(ids.size(), 1); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java index 247a1705a..4a45fa055 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java @@ -188,7 +188,7 @@ public void testRebuildIndex() { insert(); Collection indices = collection.listIndices(); for (IndexDescriptor idx : indices) { - collection.rebuildIndex(idx.getIndexFields().getFieldNames().toArray(new String[0])); + collection.rebuildIndex(idx.getFields().getFieldNames().toArray(new String[0])); } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java index fe37845e1..a59e7e459 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java @@ -134,7 +134,7 @@ public void testRebuildIndex() { insert(); Collection indices = collection.listIndices(); for (IndexDescriptor idx : indices) { - collection.rebuildIndex(idx.getIndexFields().getFieldNames().toArray(new String[0])); + collection.rebuildIndex(idx.getFields().getFieldNames().toArray(new String[0])); } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java index 0055f97ac..08cf8fc9f 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java @@ -88,6 +88,7 @@ public Document processAfterRead(Document document) { .put("expiryDate", new Date()); collection.insert(document); + cvvProcessor.process(collection); collection.addProcessor(cvvProcessor); } @@ -198,15 +199,4 @@ public void testIndexOnEncryptedField() { Document document = collection.find(where("cvv").eq("008")).firstOrNull(); assertNull(document); } - - @Test - public void testRemoveProcessor() { - Document document = collection.find(where("cvv").eq("008")).firstOrNull(); - assertNull(document); - - collection.removeProcessor(cvvProcessor); - - document = collection.find(where("cvv").eq("008")).firstOrNull(); - assertNotNull(document); - } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java index ad3534435..ee6963f91 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java @@ -18,7 +18,7 @@ package org.dizitart.no2.integration.collection; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java index 813a36503..d9b4e5816 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java @@ -17,6 +17,8 @@ package org.dizitart.no2.integration.event; +import org.dizitart.no2.collection.UpdateOptions; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.Employee; import org.dizitart.no2.Nitrite; @@ -118,6 +120,9 @@ public void setUp() { db = nitriteBuilder.openOrCreate(); } + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new Employee.EmployeeConverter()); + employeeRepository = db.getRepository(Employee.class); listener = new SampleListenerCollection(); employeeRepository.subscribe(listener); @@ -159,7 +164,7 @@ public void testUpsert() { e.setEmpId(1L); e.setAddress("abcd"); - employeeRepository.update(where("empId").eq(1), e, true); + employeeRepository.update(where("empId").eq(1), e, UpdateOptions.updateOptions(true)); await().atMost(1, TimeUnit.SECONDS).until(listenerPrepared(EventType.Insert)); assertEquals(listener.getAction(), EventType.Insert); assertNotNull(listener.getItem()); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java deleted file mode 100644 index 56ef25c22..000000000 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2017-2021 Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.dizitart.no2.integration.migrate; - -import lombok.Data; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; -import org.dizitart.no2.repository.annotations.Entity; -import org.dizitart.no2.repository.annotations.Id; -import org.dizitart.no2.repository.annotations.Index; - -/** - * @author Anindya Chatterjee - */ -@Data -@Entity(value = "new", indices = { - @Index(value = "familyName", type = IndexType.NON_UNIQUE), - @Index(value = "fullName", type = IndexType.NON_UNIQUE), - @Index(value = "literature.ratings", type = IndexType.NON_UNIQUE), -}) -public class NewClass implements Mappable { - @Id - private Long empId; - private String firstName; - private String familyName; - private String fullName; - private Literature literature; - - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("empId", empId) - .put("firstName", firstName) - .put("familyName", familyName) - .put("fullName", fullName) - .put("literature", literature.write(mapper)); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - firstName = document.get("firstName", String.class); - familyName = document.get("familyName", String.class); - fullName = document.get("fullName", String.class); - - Document doc = document.get("literature", Document.class); - literature = new Literature(); - literature.read(mapper, doc); - } - - @Data - public static class Literature implements Mappable { - private String text; - private Integer ratings; - - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("text", text) - .put("ratings", ratings); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - text = document.get("text", String.class); - ratings = document.get("ratings", Integer.class); - } - } -} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java deleted file mode 100644 index 3bbb09b4e..000000000 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2017-2021 Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.dizitart.no2.integration.migrate; - -import lombok.Data; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; -import org.dizitart.no2.repository.annotations.Entity; -import org.dizitart.no2.repository.annotations.Id; -import org.dizitart.no2.repository.annotations.Index; - -/** - * @author Anindya Chatterjee - */ -@Data -@Entity(value = "old", indices = { - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "lastName", type = IndexType.NON_UNIQUE), - @Index(value = "literature.text", type = IndexType.FULL_TEXT), - @Index(value = "literature.ratings", type = IndexType.NON_UNIQUE), -}) -public class OldClass implements Mappable { - @Id - private String uuid; - private String empId; - private String firstName; - private String lastName; - private Literature literature; - - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("empId", empId) - .put("uuid", uuid) - .put("firstName", firstName) - .put("lastName", lastName) - .put("literature", literature.write(mapper)); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", String.class); - uuid = document.get("uuid", String.class); - firstName = document.get("firstName", String.class); - lastName = document.get("lastName", String.class); - - Document doc = document.get("literature", Document.class); - literature = new Literature(); - literature.read(mapper, doc); - } - - @Data - public static class Literature implements Mappable { - private String text; - private Float ratings; - - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("text", text) - .put("ratings", ratings); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - text = document.get("text", String.class); - ratings = document.get("ratings", Float.class); - } - } -} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/MigrationTest.java similarity index 89% rename from nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java rename to nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/MigrationTest.java index 9b4eeab29..2843ba4c5 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/MigrationTest.java @@ -15,9 +15,10 @@ * */ -package org.dizitart.no2.integration.migrate; +package org.dizitart.no2.integration.migration; import com.github.javafaker.Faker; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.Document; @@ -27,7 +28,7 @@ import org.dizitart.no2.exceptions.MigrationException; import org.dizitart.no2.index.IndexOptions; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.migration.Instructions; +import org.dizitart.no2.migration.InstructionSet; import org.dizitart.no2.migration.Migration; import org.dizitart.no2.migration.TypeConverter; import org.dizitart.no2.mvstore.MVStoreModule; @@ -57,6 +58,12 @@ public class MigrationTest { @Before public void setUp() { db = createDb(dbPath); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new OldClass.Converter()); + documentMapper.registerEntityConverter(new OldClass.Literature.Converter()); + documentMapper.registerEntityConverter(new NewClass.Converter()); + documentMapper.registerEntityConverter(new NewClass.Literature.Converter()); + faker = new Faker(); } @@ -91,7 +98,7 @@ public void testRepositoryMigrate() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forDatabase() .addPassword("test-user", "test-password"); @@ -121,10 +128,16 @@ public void migrate(Instructions instruction) { .addMigrations(migration) .openOrCreate("test-user", "test-password"); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new OldClass.Converter()); + documentMapper.registerEntityConverter(new OldClass.Literature.Converter()); + documentMapper.registerEntityConverter(new NewClass.Converter()); + documentMapper.registerEntityConverter(new NewClass.Literature.Converter()); + ObjectRepository newRepo = db.getRepository(NewClass.class); assertEquals(newRepo.size(), 10); assertTrue(db.listCollectionNames().isEmpty()); - assertTrue(db.listKeyedRepository().isEmpty()); + assertTrue(db.listKeyedRepositories().isEmpty()); assertEquals((int) db.getDatabaseMetaData().getSchemaVersion(), 2); } @@ -148,7 +161,7 @@ public void testCollectionMigrate() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forDatabase() .addPassword("test-user", "test-password"); @@ -179,11 +192,11 @@ public void migrate(Instructions instruction) { migration = new Migration(2, 3) { @Override - public void migrate(Instructions instructions) { - instructions.forDatabase() + public void migrate(InstructionSet instructionSet) { + instructionSet.forDatabase() .changePassword("test-user", "test-password", "password"); - instructions.forCollection("testCollectionMigrate") + instructionSet.forCollection("testCollectionMigrate") .dropIndex("firstName") .deleteField("bloodGroup") .addField("name", document -> faker.name().fullName()) @@ -230,7 +243,7 @@ public void testOpenWithoutSchemaVersion() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testOpenWithoutSchemaVersion") @@ -282,7 +295,7 @@ public void testDescendingSchema() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testDescendingSchema") @@ -308,9 +321,9 @@ public void migrate(Instructions instruction) { migration = new Migration(2, Constants.INITIAL_SCHEMA_VERSION) { @Override - public void migrate(Instructions instructions) { + public void migrate(InstructionSet instructionSet) { - instructions.forCollection("testDescendingSchema") + instructionSet.forCollection("testDescendingSchema") .rename("test"); } }; @@ -346,7 +359,7 @@ public void testMigrationWithoutVersion() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testMigrationWithoutVersion") @@ -387,7 +400,7 @@ public void testWrongSchemaVersionNoMigration() { Migration migration = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testWrongSchemaVersionNoMigration") .rename("test") @@ -416,8 +429,8 @@ public void migrate(Instructions instruction) { migration = new Migration(2, 3) { @Override - public void migrate(Instructions instructions) { - instructions.forCollection("test") + public void migrate(InstructionSet instructionSet) { + instructionSet.forCollection("test") .rename("testWrongSchemaVersionNoMigration"); } }; @@ -456,7 +469,7 @@ public void testReOpenAfterMigration() { Migration migration = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testReOpenAfterMigration") .rename("test") @@ -524,7 +537,7 @@ public void testMultipleMigrations() { Migration migration1 = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testMultipleMigrations") .rename("test"); @@ -533,7 +546,7 @@ public void migrate(Instructions instruction) { Migration migration2 = new Migration(2, 3) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .addField("fullName", "Dummy Name"); } @@ -558,7 +571,7 @@ public void migrate(Instructions instruction) { Migration migration3 = new Migration(3, 4) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .addField("age", 10); } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/NewClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/NewClass.java new file mode 100644 index 000000000..177023c41 --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/NewClass.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.migration; + +import lombok.Data; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; +import org.dizitart.no2.repository.annotations.Entity; +import org.dizitart.no2.repository.annotations.Id; +import org.dizitart.no2.repository.annotations.Index; + +/** + * @author Anindya Chatterjee + */ +@Data +@Entity(value = "new", indices = { + @Index(fields = "familyName", type = IndexType.NON_UNIQUE), + @Index(fields = "fullName", type = IndexType.NON_UNIQUE), + @Index(fields = "literature.ratings", type = IndexType.NON_UNIQUE), +}) +public class NewClass { + @Id + private Long empId; + private String firstName; + private String familyName; + private String fullName; + private Literature literature; + + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return NewClass.class; + } + + @Override + public Document toDocument(NewClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument("empId", entity.empId) + .put("firstName", entity.firstName) + .put("familyName", entity.familyName) + .put("fullName", entity.fullName) + .put("literature", nitriteMapper.convert(entity.literature, Document.class)); + } + + @Override + public NewClass fromDocument(Document document, NitriteMapper nitriteMapper) { + NewClass entity = new NewClass(); + entity.empId = document.get("empId", Long.class); + entity.firstName = document.get("firstName", String.class); + entity.familyName = document.get("familyName", String.class); + entity.fullName = document.get("fullName", String.class); + + Document doc = document.get("literature", Document.class); + entity.literature = nitriteMapper.convert(doc, Literature.class); + return entity; + } + } + + @Data + public static class Literature { + private String text; + private Integer ratings; + + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return Literature.class; + } + + @Override + public Document toDocument(Literature entity, NitriteMapper nitriteMapper) { + return Document.createDocument("text", entity.text) + .put("ratings", entity.ratings); + } + + @Override + public Literature fromDocument(Document document, NitriteMapper nitriteMapper) { + Literature entity = new Literature(); + entity.text = document.get("text", String.class); + entity.ratings = document.get("ratings", Integer.class); + return entity; + } + } + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/OldClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/OldClass.java new file mode 100644 index 000000000..cd7104dd3 --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/migration/OldClass.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.migration; + +import lombok.Data; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; +import org.dizitart.no2.repository.annotations.Entity; +import org.dizitart.no2.repository.annotations.Id; +import org.dizitart.no2.repository.annotations.Index; + +/** + * @author Anindya Chatterjee + */ +@Data +@Entity(value = "old", indices = { + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "lastName", type = IndexType.NON_UNIQUE), + @Index(fields = "literature.text", type = IndexType.FULL_TEXT), + @Index(fields = "literature.ratings", type = IndexType.NON_UNIQUE), +}) +public class OldClass { + @Id + private String uuid; + private String empId; + private String firstName; + private String lastName; + private Literature literature; + + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return OldClass.class; + } + + @Override + public Document toDocument(OldClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument("empId", entity.empId) + .put("uuid", entity.uuid) + .put("firstName", entity.firstName) + .put("lastName", entity.lastName) + .put("literature", nitriteMapper.convert(entity.literature, Document.class)); + } + + @Override + public OldClass fromDocument(Document document, NitriteMapper nitriteMapper) { + OldClass entity = new OldClass(); + entity.empId = document.get("empId", String.class); + entity.uuid = document.get("uuid", String.class); + entity.firstName = document.get("firstName", String.class); + entity.lastName = document.get("lastName", String.class); + + Document doc = document.get("literature", Document.class); + entity.literature = nitriteMapper.convert(doc, Literature.class); + return entity; + } + } + + @Data + public static class Literature { + private String text; + private Float ratings; + + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return Literature.class; + } + + @Override + public Document toDocument(Literature entity, NitriteMapper nitriteMapper) { + return Document.createDocument("text", entity.text) + .put("ratings", entity.ratings); + } + + @Override + public Literature fromDocument(Document document, NitriteMapper nitriteMapper) { + Literature entity = new Literature(); + entity.text = document.get("text", String.class); + entity.ratings = document.get("ratings", Float.class); + return entity; + } + } + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java index 2d33d0c43..f279620f0 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java @@ -17,10 +17,16 @@ package org.dizitart.no2.integration.repository; -import org.dizitart.no2.integration.Retry; -import org.dizitart.no2.integration.repository.data.*; import org.dizitart.no2.Nitrite; import org.dizitart.no2.NitriteBuilder; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; +import org.dizitart.no2.integration.Retry; +import org.dizitart.no2.integration.repository.data.*; +import org.dizitart.no2.integration.repository.decorator.ManufacturerConverter; +import org.dizitart.no2.integration.repository.decorator.MiniProduct; +import org.dizitart.no2.integration.repository.decorator.ProductConverter; +import org.dizitart.no2.integration.repository.decorator.ProductIdConverter; +import org.dizitart.no2.integration.transaction.TxData; import org.dizitart.no2.mvstore.MVStoreModule; import org.dizitart.no2.mvstore.MVStoreModuleBuilder; import org.dizitart.no2.repository.ObjectRepository; @@ -135,6 +141,33 @@ protected void openDb() { } else { db = nitriteBuilder.openOrCreate(); } + + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new RepositoryJoinTest.Person.Converter()); + documentMapper.registerEntityConverter(new RepositoryJoinTest.Address.Converter()); + documentMapper.registerEntityConverter(new RepositoryJoinTest.PersonDetails.Converter()); + documentMapper.registerEntityConverter(new Company.CompanyConverter()); + documentMapper.registerEntityConverter(new Employee.EmployeeConverter()); + documentMapper.registerEntityConverter(new Note.NoteConverter()); + documentMapper.registerEntityConverter(new Book.BookConverter()); + documentMapper.registerEntityConverter(new BookId.BookIdConverter()); + documentMapper.registerEntityConverter(new ClassA.ClassAConverter()); + documentMapper.registerEntityConverter(new ClassBConverter()); + documentMapper.registerEntityConverter(new ClassC.ClassCConverter()); + documentMapper.registerEntityConverter(new ElemMatch.Converter()); + documentMapper.registerEntityConverter(new InternalClass.Converter()); + documentMapper.registerEntityConverter(new UniversalTextTokenizerTest.TextData.Converter()); + documentMapper.registerEntityConverter(new SubEmployee.Converter()); + documentMapper.registerEntityConverter(new ProductScore.Converter()); + documentMapper.registerEntityConverter(new PersonEntity.Converter()); + documentMapper.registerEntityConverter(new RepeatableIndexTest.Converter()); + documentMapper.registerEntityConverter(new EncryptedPerson.Converter()); + documentMapper.registerEntityConverter(new TxData.Converter()); + documentMapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter()); + documentMapper.registerEntityConverter(new ProductConverter()); + documentMapper.registerEntityConverter(new ProductIdConverter()); + documentMapper.registerEntityConverter(new ManufacturerConverter()); + documentMapper.registerEntityConverter(new MiniProduct.Converter()); } @After diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java index 965a8d8ab..42c80d4c2 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java @@ -21,6 +21,8 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.Company; import org.dizitart.no2.integration.repository.data.Note; @@ -28,7 +30,6 @@ import org.dizitart.no2.NitriteConfig; import org.dizitart.no2.collection.Document; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.mvstore.MVStoreModule; import org.dizitart.no2.repository.ObjectRepository; @@ -71,6 +72,11 @@ public void setUp() { .fieldSeparator(":") .openOrCreate(); + SimpleDocumentMapper mapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + mapper.registerEntityConverter(new Company.CompanyConverter()); + mapper.registerEntityConverter(new EmployeeForCustomSeparator.EmployeeForCustomSeparatorConverter()); + mapper.registerEntityConverter(new Note.NoteConverter()); + repository = db.getRepository(EmployeeForCustomSeparator.class); } @@ -118,11 +124,11 @@ public void testFindByEmbeddedField() { @ToString @EqualsAndHashCode @Indices({ - @Index(value = "joinDate", type = IndexType.NON_UNIQUE), - @Index(value = "address", type = IndexType.FULL_TEXT), - @Index(value = "employeeNote:text", type = IndexType.FULL_TEXT) + @Index(fields = "joinDate", type = IndexType.NON_UNIQUE), + @Index(fields = "address", type = IndexType.FULL_TEXT), + @Index(fields = "employeeNote:text", type = IndexType.FULL_TEXT) }) - public static class EmployeeForCustomSeparator implements Serializable, Mappable { + public static class EmployeeForCustomSeparator implements Serializable { @Id @Getter @Setter @@ -160,28 +166,39 @@ public EmployeeForCustomSeparator(EmployeeForCustomSeparator copy) { employeeNote = copy.employeeNote; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument().put("empId", empId) - .put("joinDate", joinDate) - .put("address", address) - .put("blob", blob) - .put("company", company.write(mapper)) - .put("employeeNote", employeeNote.write(mapper)); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - joinDate = document.get("joinDate", Date.class); - address = document.get("address", String.class); - blob = document.get("blob", byte[].class); - employeeNote = new Note(); - Document doc = document.get("employeeNote", Document.class); - employeeNote.read(mapper, doc); - company = new Company(); - doc = document.get("company", Document.class); - company.read(mapper, doc); + public static class EmployeeForCustomSeparatorConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return EmployeeForCustomSeparator.class; + } + + @Override + public Document toDocument(EmployeeForCustomSeparator entity, NitriteMapper nitriteMapper) { + return Document.createDocument().put("empId", entity.empId) + .put("joinDate", entity.joinDate) + .put("address", entity.address) + .put("blob", entity.blob) + .put("company", nitriteMapper.convert(entity.company, Document.class)) + .put("employeeNote", nitriteMapper.convert(entity.employeeNote, Document.class)); + } + + @Override + public EmployeeForCustomSeparator fromDocument(Document document, NitriteMapper nitriteMapper) { + EmployeeForCustomSeparator entity = new EmployeeForCustomSeparator(); + + entity.empId = document.get("empId", Long.class); + entity.joinDate = document.get("joinDate", Date.class); + entity.address = document.get("address", String.class); + entity.blob = document.get("blob", byte[].class); + + Document doc = document.get("employeeNote", Document.class); + entity.employeeNote = nitriteMapper.convert(doc, Note.class); + + doc = document.get("company", Document.class); + entity.company = nitriteMapper.convert(doc, Company.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java index 73a093082..8f31d6408 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java @@ -63,6 +63,10 @@ public void setUp() { persons.insert(person); + // process existing data + fieldProcessor.process(persons); + + // add for further changes persons.addProcessor(fieldProcessor); person = new EncryptedPerson(); @@ -187,21 +191,4 @@ public void testIndexOnEncryptedField() { EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull(); assertNull(person); } - - @Test - public void testRemoveProcessor() { - EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull(); - assertNull(person); - - person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull(); - assertNull(person); - - persons.removeProcessor(fieldProcessor); - - person = persons.find(where("cvv").eq("008")).firstOrNull(); - assertNotNull(person); - - person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull(); - assertNotNull(person); - } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java index 1a52708ce..e6a2a58d4 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -27,20 +27,29 @@ * @author Anindya Chatterjee. */ @Data -class InternalClass implements Mappable { +class InternalClass { @Id - private long id; + private Long id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("name", name); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return InternalClass.class; + } + + @Override + public Document toDocument(InternalClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - name = document.get("name", String.class); + @Override + public InternalClass fromDocument(Document document, NitriteMapper nitriteMapper) { + InternalClass entity = new InternalClass(); + entity.id = document.get("id", Long.class); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java index dedd5e256..4b776ea17 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java @@ -17,6 +17,7 @@ package org.dizitart.no2.integration.repository; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.WithNitriteId; import org.dizitart.no2.Nitrite; @@ -53,6 +54,9 @@ public class NitriteIdAsIdTest { @Before public void before() { db = TestUtil.createDb(fileName); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter()); + repo = db.getRepository(WithNitriteId.class); } @@ -90,7 +94,7 @@ public void testNitriteIdField() { } @Test(expected = InvalidIdException.class) - public void setIdDuringInsert() { + public void testSetIdDuringInsert() { WithNitriteId item1 = new WithNitriteId(); item1.name = "first"; item1.idField = NitriteId.newId(); @@ -99,7 +103,7 @@ public void setIdDuringInsert() { } @Test - public void changeIdDuringUpdate() { + public void testChangeIdDuringUpdate() { WithNitriteId item2 = new WithNitriteId(); item2.name = "second"; WriteResult result = repo.insert(item2); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java index e070d485d..66a816959 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java @@ -17,6 +17,7 @@ package org.dizitart.no2.integration.repository; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.*; import org.dizitart.no2.Nitrite; @@ -50,6 +51,18 @@ public class ObjectRepositoryNegativeTest { @Before public void setUp() { db = TestUtil.createDb(dbPath); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new WithPublicField.Converter()); + documentMapper.registerEntityConverter(new WithObjectId.Converter()); + documentMapper.registerEntityConverter(new WithOutId.Converter()); + documentMapper.registerEntityConverter(new WithoutEmbeddedId.Converter()); + documentMapper.registerEntityConverter(new WithoutEmbeddedId.NestedId.Converter()); + documentMapper.registerEntityConverter(new WithEmptyStringId.Converter()); + documentMapper.registerEntityConverter(new WithNullId.Converter()); + documentMapper.registerEntityConverter(new Employee.EmployeeConverter()); + documentMapper.registerEntityConverter(new Company.CompanyConverter()); + documentMapper.registerEntityConverter(new Note.NoteConverter()); + documentMapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter()); } @After @@ -59,7 +72,7 @@ public void close() throws IOException { deleteDb(dbPath); } - @Test(expected = ObjectMappingException.class) + @Test(expected = ValidationException.class) public void testWithCircularReference() { ObjectRepository repository = db.getRepository(WithCircularReference.class); @@ -79,7 +92,7 @@ public void testWithCircularReference() { } } - @Test(expected = ObjectMappingException.class) + @Test(expected = ValidationException.class) public void testWithCustomConstructor() { ObjectRepository repository = db.getRepository(WithCustomConstructor.class); @@ -137,7 +150,7 @@ public void testWithObjectId() { ObjectRepository repository = db.getRepository(WithObjectId.class); WithOutId id = new WithOutId(); id.setName("test"); - id.setNumber(1); + id.setNumber(1L); WithObjectId object = new WithObjectId(); object.setWithOutId(id); @@ -185,7 +198,7 @@ public void testGetByNullId() { ObjectRepository repository = db.getRepository(WithPublicField.class); WithPublicField object = new WithPublicField(); object.name = "test"; - object.number = 2; + object.number = 2L; repository.insert(object); WithPublicField instance = repository.getById(null); @@ -221,7 +234,7 @@ public void testGetByWrongIdType() { ObjectRepository repository = db.getRepository(WithPublicField.class); WithPublicField object = new WithPublicField(); object.name = "test"; - object.number = 2; + object.number = 2L; repository.insert(object); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java index 376d4b9c4..d66ef2d9c 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java @@ -19,17 +19,21 @@ import com.github.javafaker.Faker; import lombok.Data; -import org.dizitart.no2.integration.Retry; -import org.dizitart.no2.integration.repository.data.*; import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.MappableMapper; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.exceptions.ValidationException; import org.dizitart.no2.index.IndexType; +import org.dizitart.no2.integration.Retry; +import org.dizitart.no2.integration.repository.data.*; +import org.dizitart.no2.integration.repository.decorator.ManufacturerConverter; +import org.dizitart.no2.integration.repository.decorator.MiniProduct; +import org.dizitart.no2.integration.repository.decorator.ProductConverter; +import org.dizitart.no2.integration.repository.decorator.ProductIdConverter; import org.dizitart.no2.mvstore.MVStoreModule; import org.dizitart.no2.repository.Cursor; import org.dizitart.no2.repository.ObjectRepository; @@ -66,7 +70,25 @@ public class ObjectRepositoryTest { @Before public void setUp() { - NitriteMapper mapper = new MappableMapper(); + SimpleDocumentMapper mapper = new SimpleDocumentMapper(); + mapper.registerEntityConverter(new InternalClass.Converter()); + mapper.registerEntityConverter(new EmployeeEntity.Converter()); + mapper.registerEntityConverter(new StressRecord.Converter()); + mapper.registerEntityConverter(new WithClassField.Converter()); + mapper.registerEntityConverter(new WithDateId.Converter()); + mapper.registerEntityConverter(new WithTransientField.Converter()); + mapper.registerEntityConverter(new WithOutId.Converter()); + mapper.registerEntityConverter(new ChildClass.Converter()); + mapper.registerEntityConverter(new WithOutGetterSetter.Converter()); + mapper.registerEntityConverter(new WithPrivateConstructor.Converter()); + mapper.registerEntityConverter(new WithPublicField.Converter()); + mapper.registerEntityConverter(new Employee.EmployeeConverter()); + mapper.registerEntityConverter(new Company.CompanyConverter()); + mapper.registerEntityConverter(new ProductConverter()); + mapper.registerEntityConverter(new ProductIdConverter()); + mapper.registerEntityConverter(new ManufacturerConverter()); + mapper.registerEntityConverter(new MiniProduct.Converter()); + MVStoreModule storeModule = MVStoreModule.withConfig() .filePath(dbPath) .build(); @@ -115,7 +137,7 @@ public void testWithOutId() { ObjectRepository repository = db.getRepository(WithOutId.class); WithOutId object = new WithOutId(); object.setName("test"); - object.setNumber(2); + object.setNumber(2L); repository.insert(object); for (WithOutId instance : repository.find()) { @@ -129,7 +151,7 @@ public void testWithPublicField() { ObjectRepository repository = db.getRepository(WithPublicField.class); WithPublicField object = new WithPublicField(); object.name = "test"; - object.number = 2; + object.number = 2L; repository.insert(object); WithPublicField instance = repository.getById("test"); @@ -141,7 +163,7 @@ public void testWithPublicField() { public void testWithTransientField() { ObjectRepository repository = db.getRepository(WithTransientField.class); WithTransientField object = new WithTransientField(); - object.setNumber(2); + object.setNumber(2L); object.setName("test"); repository.insert(object); @@ -180,7 +202,7 @@ public void testWriteThousandRecords() { public void testWithPackagePrivateClass() { ObjectRepository repository = db.getRepository(InternalClass.class); InternalClass internalClass = new InternalClass(); - internalClass.setId(1); + internalClass.setId(1L); internalClass.setName("name"); repository.insert(internalClass); @@ -293,7 +315,7 @@ public void testKeyedRepository() { assertTrue(db.hasRepository(Employee.class, "developers")); assertEquals(db.listRepositories().size(), 1); - assertEquals(db.listKeyedRepository().size(), 2); + assertEquals(db.listKeyedRepositories().size(), 2); assertEquals(employeeRepo.find(where("address").text("abcd")).size(), 1); assertEquals(employeeRepo.find(where("address").text("xyz")).size(), 1); @@ -322,7 +344,7 @@ public void testEntityRepository() { assertTrue(errored); assertTrue(db.listRepositories().contains("entity.employee")); - assertEquals(db.listKeyedRepository().size(), 2); + assertEquals(db.listKeyedRepositories().size(), 2); assertEquals(db.listCollectionNames().size(), 0); assertTrue(managerRepo.hasIndex("firstName")); @@ -331,7 +353,7 @@ public void testEntityRepository() { assertTrue(employeeRepo.hasIndex("lastName")); managerRepo.drop(); - assertEquals(db.listKeyedRepository().size(), 1); + assertEquals(db.listKeyedRepositories().size(), 1); } @Test @@ -347,10 +369,10 @@ public void testIssue217() { @Data @Entity(value = "entity.employee", indices = { - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "lastName", type = IndexType.NON_UNIQUE), + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "lastName", type = IndexType.NON_UNIQUE), }) - private static class EmployeeEntity implements Mappable { + private static class EmployeeEntity { private static final Faker faker = new Faker(); @Id @@ -364,18 +386,28 @@ public EmployeeEntity() { lastName = faker.name().lastName(); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("firstName", firstName) - .put("lastName", lastName); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - firstName = document.get("firstName", String.class); - lastName = document.get("lastName", String.class); + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return EmployeeEntity.class; + } + + @Override + public Document toDocument(EmployeeEntity entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("firstName", entity.firstName) + .put("lastName", entity.lastName); + } + + @Override + public EmployeeEntity fromDocument(Document document, NitriteMapper nitriteMapper) { + EmployeeEntity entity = new EmployeeEntity(); + entity.id = document.get("id", Long.class); + entity.firstName = document.get("firstName", String.class); + entity.lastName = document.get("lastName", String.class); + return entity; + } } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java index d617767c0..68a2371b3 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java @@ -20,7 +20,7 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.*; import org.dizitart.no2.collection.events.CollectionEventListener; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.common.WriteResult; import org.dizitart.no2.common.concurrent.LockService; import org.dizitart.no2.common.processors.Processor; @@ -66,10 +66,10 @@ public void testRepositoryFactory() { @Test(expected = ValidationException.class) public void testNullType() { RepositoryFactory factory = new RepositoryFactory(new CollectionFactory(new LockService())); - factory.getRepository(db.getConfig(), null, "dummy"); + factory.getRepository(db.getConfig(), (Class) null, "dummy"); } - @Test + @Test(expected = ValidationException.class) public void testNullCollection() { RepositoryFactory factory = new RepositoryFactory(new CollectionFactory(new LockService())); factory.getRepository(db.getConfig(), DummyCollection.class, null); @@ -136,11 +136,6 @@ public void addProcessor(Processor processor) { } - @Override - public void removeProcessor(Processor processor) { - - } - @Override public void createIndex(IndexOptions indexOptions, String... fields) { diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java index 4dacdbff8..05634377f 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java @@ -22,7 +22,7 @@ import org.dizitart.no2.collection.NitriteId; import org.dizitart.no2.common.Lookup; import org.dizitart.no2.common.RecordStream; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.repository.ObjectRepository; @@ -31,10 +31,7 @@ import org.junit.Before; import org.junit.Test; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; +import java.util.*; import static org.dizitart.no2.collection.Document.createDocument; import static org.dizitart.no2.collection.FindOptions.skipBy; @@ -138,80 +135,120 @@ public void testRemove() { } @Data - public static class Person implements Mappable { + public static class Person { @Id private NitriteId nitriteId; private String id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return createDocument() - .put("nitriteId", nitriteId) - .put("id", id) - .put("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return Person.class; + } + + @Override + public Document toDocument(Person entity, NitriteMapper nitriteMapper) { + return createDocument() + .put("nitriteId", entity.nitriteId) + .put("id", entity.id) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - nitriteId = document.get("nitriteId", NitriteId.class); - id = document.get("id", String.class); - name = document.get("name", String.class); + @Override + public Person fromDocument(Document document, NitriteMapper nitriteMapper) { + Person entity = new Person(); + entity.nitriteId = document.get("nitriteId", NitriteId.class); + entity.id = document.get("id", String.class); + entity.name = document.get("name", String.class); + return entity; + } } } @Data - public static class Address implements Mappable { + public static class Address { @Id private NitriteId nitriteId; private String personId; private String street; - @Override - public Document write(NitriteMapper mapper) { - return createDocument() - .put("nitriteId", nitriteId) - .put("personId", personId) - .put("street", street); - } + public static class Converter implements EntityConverter
{ + + @Override + public Class
getEntityType() { + return Address.class; + } + + @Override + public Document toDocument(Address entity, NitriteMapper nitriteMapper) { + return createDocument() + .put("nitriteId", entity.nitriteId) + .put("personId", entity.personId) + .put("street", entity.street); + } - @Override - public void read(NitriteMapper mapper, Document document) { - nitriteId = document.get("nitriteId", NitriteId.class); - personId = document.get("personId", String.class); - street = document.get("street", String.class); + @Override + public Address fromDocument(Document document, NitriteMapper nitriteMapper) { + Address entity = new Address(); + entity.nitriteId = document.get("nitriteId", NitriteId.class); + entity.personId = document.get("personId", String.class); + entity.street = document.get("street", String.class); + return entity; + } } } @Data - public static class PersonDetails implements Mappable { + public static class PersonDetails { @Id private NitriteId nitriteId; private String id; private String name; private List
addresses; - @Override - public Document write(NitriteMapper mapper) { - return createDocument() - .put("nitriteId", nitriteId) - .put("personId", id) - .put("street", name) - .put("addresses", addresses); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return PersonDetails.class; + } + + @Override + public Document toDocument(PersonDetails entity, NitriteMapper nitriteMapper) { + List documents = new ArrayList<>(); + if (entity.addresses != null) { + for (Address address : entity.addresses) { + documents.add(nitriteMapper.convert(address, Document.class)); + } + } + + return createDocument() + .put("nitriteId", entity.nitriteId) + .put("personId", entity.id) + .put("street", entity.name) + .put("addresses", documents); + } + + @Override + public PersonDetails fromDocument(Document document, NitriteMapper nitriteMapper) { + PersonDetails entity = new PersonDetails(); + + entity.nitriteId = document.get("nitriteId", NitriteId.class); + entity.id = document.get("id", String.class); + entity.name = document.get("name", String.class); + + Collection documents = document.get("addresses", Collection.class); + if (documents != null) { + entity.addresses = new ArrayList<>(); + for (Document doc : documents) { + Address address = nitriteMapper.convert(doc, Address.class); + entity.addresses.add(address); + } + } - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - nitriteId = document.get("nitriteId", NitriteId.class); - id = document.get("id", String.class); - name = document.get("name", String.class); - Set documents = document.get("addresses", Set.class); - this.addresses = new ArrayList<>(); - for (Document doc : documents) { - Address address = new Address(); - address.read(mapper, doc); - addresses.add(address); + return entity; } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java index d5bd0f5cb..f12c86fdf 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java @@ -39,6 +39,7 @@ import static org.awaitility.Awaitility.await; import static org.dizitart.no2.collection.Document.createDocument; +import static org.dizitart.no2.collection.UpdateOptions.updateOptions; import static org.dizitart.no2.filters.FluentFilter.where; import static org.junit.Assert.*; @@ -210,7 +211,7 @@ public void testUpsertTrue() { employee.setEmployeeNote(empNote1); WriteResult writeResult - = employeeRepository.update(where("empId").eq(12), employee, true); + = employeeRepository.update(where("empId").eq(12), employee, updateOptions(true)); assertEquals(writeResult.getAffectedCount(), 1); result = employeeRepository.find(where("joinDate").eq(joiningDate)); @@ -235,7 +236,7 @@ public void testUpsertFalse() { employee.setEmployeeNote(empNote1); WriteResult writeResult - = employeeRepository.update(where("empId").eq(12), employee, false); + = employeeRepository.update(where("empId").eq(12), employee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 0); result = employeeRepository.find(where("joinDate").eq(joiningDate)); @@ -327,7 +328,7 @@ public void testMultiUpdateWithObject() { update.setAddress("new address"); WriteResult writeResult - = employeeRepository.update(where("joinDate").eq(now), update, false); + = employeeRepository.update(where("joinDate").eq(now), update, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 0); } @@ -361,7 +362,7 @@ public void testUpdateWithChangedId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); assertEquals(count, employeeRepository.size()); @@ -380,7 +381,7 @@ public void testUpdateWithNullId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); } @@ -396,7 +397,7 @@ public void testUpdateWithDuplicateId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); assertEquals(count, employeeRepository.size()); @@ -563,12 +564,12 @@ public void testDelete() { public void testUpdateObjectNotExistsUpsertTrue() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(2); + a.setId(2L); a.setName("second"); // it will insert as new object @@ -580,18 +581,18 @@ public void testUpdateObjectNotExistsUpsertTrue() { public void testUpdateObjectNotExistsUpsertFalse() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(2); + a.setId(2L); a.setName("second"); // no changes will happen to repository repo.update(a, false); assertEquals(repo.size(), 1); - assertEquals(repo.find().firstOrNull().getId(), 1); + assertEquals(repo.find().firstOrNull().getId().longValue(), 1); assertEquals(repo.find().firstOrNull().getName(), "first"); } @@ -599,18 +600,18 @@ public void testUpdateObjectNotExistsUpsertFalse() { public void testUpdateObjectExistsUpsertTrue() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("second"); // update existing object, keep id same repo.update(a, true); assertEquals(repo.size(), 1); - assertEquals(repo.find().firstOrNull().getId(), 1); + assertEquals(repo.find().firstOrNull().getId().longValue(), 1); assertEquals(repo.find().firstOrNull().getName(), "second"); } @@ -618,18 +619,18 @@ public void testUpdateObjectExistsUpsertTrue() { public void testUpdateObjectExistsUpsertFalse() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("second"); // update existing object, keep id same repo.update(a, false); assertEquals(repo.size(), 1); - assertEquals(repo.find().firstOrNull().getId(), 1); + assertEquals(repo.find().firstOrNull().getId().longValue(), 1); assertEquals(repo.find().firstOrNull().getName(), "second"); } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java index 68015c9f3..47a798519 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java @@ -18,10 +18,11 @@ package org.dizitart.no2.integration.repository; import lombok.Getter; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.repository.data.*; import org.dizitart.no2.collection.Document; import org.dizitart.no2.common.SortOrder; -import org.dizitart.no2.common.mapper.Mappable; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.exceptions.FilterException; import org.dizitart.no2.exceptions.InvalidIdException; @@ -364,21 +365,20 @@ public void testElemMatchFilter() { final ProductScore score6 = new ProductScore("xyz", 8); ObjectRepository repository = db.getRepository(ElemMatch.class); - ElemMatch e1 = new ElemMatch() {{ - setId(1); - setStrArray(new String[]{"a", "b"}); - setProductScores(new ProductScore[]{score1, score4}); - }}; - ElemMatch e2 = new ElemMatch() {{ - setId(2); - setStrArray(new String[]{"d", "e"}); - setProductScores(new ProductScore[]{score2, score5}); - }}; - ElemMatch e3 = new ElemMatch() {{ - setId(3); - setStrArray(new String[]{"a", "f"}); - setProductScores(new ProductScore[]{score3, score6}); - }}; + ElemMatch e1 = new ElemMatch(); + e1.setId(1L); + e1.setStrArray(new String[]{"a", "b"}); + e1.setProductScores(new ProductScore[]{score1, score4}); + + ElemMatch e2 = new ElemMatch(); + e2.setId(2L); + e2.setStrArray(new String[]{"d", "e"}); + e2.setProductScores(new ProductScore[]{score2, score5}); + + ElemMatch e3 = new ElemMatch(); + e3.setId(3L); + e3.setStrArray(new String[]{"a", "f"}); + e3.setProductScores(new ProductScore[]{score3, score6}); repository.insert(e1, e2, e3); @@ -563,24 +563,37 @@ public void testIdSet() { @Test public void testBetweenFilter() { @Getter - class TestData implements Mappable { + class TestData { private Date age; public TestData(Date age) { this.age = age; } + } + + class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return TestData.class; + } @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("age", age); + public Document toDocument(TestData entity, NitriteMapper nitriteMapper) { + return Document.createDocument("age", entity.age); } @Override - public void read(NitriteMapper mapper, Document document) { - age = document.get("age", Date.class); + public TestData fromDocument(Document document, NitriteMapper nitriteMapper) { + TestData entity = new TestData(new Date()); + entity.age = document.get("age", Date.class); + return entity; } } + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new Converter()); + TestData data1 = new TestData(new GregorianCalendar(2020, Calendar.JANUARY, 11).getTime()); TestData data2 = new TestData(new GregorianCalendar(2021, Calendar.FEBRUARY, 12).getTime()); TestData data3 = new TestData(new GregorianCalendar(2022, Calendar.MARCH, 13).getTime()); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java index 057a0b78e..59d6681ba 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java @@ -20,8 +20,9 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.NitriteBuilder; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.index.NitriteTextIndexer; import org.dizitart.no2.index.fulltext.Languages; @@ -55,6 +56,8 @@ public class UniversalTextTokenizerTest extends BaseObjectRepositoryTest { @Override public void setUp() { openDb(); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TextData.Converter()); textRepository = db.getRepository(TextData.class); @@ -181,22 +184,31 @@ public void testUniversalFullTextIndexing() { } @Indices( - @Index(value = "text", type = IndexType.FULL_TEXT) + @Index(fields = "text", type = IndexType.FULL_TEXT) ) - public static class TextData implements Mappable { - public int id; + public static class TextData { + public Integer id; public String text; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("text", text); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return TextData.class; + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Integer.class); - text = document.get("text", String.class); + @Override + public Document toDocument(TextData entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("text", entity.text); + } + + @Override + public TextData fromDocument(Document document, NitriteMapper nitriteMapper) { + TextData entity = new TextData(); + entity.id = document.get("id", Integer.class); + entity.text = document.get("text", String.class); + return entity; + } } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java index 5a147769c..c27993fc4 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Entity; @@ -35,12 +35,12 @@ */ @Data @Entity(value = "books", indices = { - @Index(value = "tags", type = IndexType.NON_UNIQUE), - @Index(value = "description", type = IndexType.FULL_TEXT), - @Index(value = { "price", "publisher" }) + @Index(fields = "tags", type = IndexType.NON_UNIQUE), + @Index(fields = "description", type = IndexType.FULL_TEXT), + @Index(fields = { "price", "publisher" }) }) -public class Book implements Mappable { - @Id(fieldName = "book_id") +public class Book { + @Id(fieldName = "book_id", embeddedFields = { "isbn", "book_name" }) private BookId bookId; private String publisher; @@ -51,22 +51,32 @@ public class Book implements Mappable { private String description; - @Override - public Document write(NitriteMapper mapper) { - return createDocument("book_id", mapper.convert(bookId, Document.class)) - .put("publisher", publisher) - .put("price", price) - .put("tags", tags) - .put("description", description); - } + public static class BookConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Book.class; + } + + @Override + public Document toDocument(Book entity, NitriteMapper nitriteMapper) { + return createDocument("book_id", nitriteMapper.convert(entity.bookId, Document.class)) + .put("publisher", entity.publisher) + .put("price", entity.price) + .put("tags", entity.tags) + .put("description", entity.description); + } - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - bookId = mapper.convert(document.get("book_id"), BookId.class); - publisher = document.get("publisher", String.class); - price = document.get("price", Double.class); - tags = (List) document.get("tags", List.class); - description = document.get("description", String.class); + @Override + @SuppressWarnings("unchecked") + public Book fromDocument(Document document, NitriteMapper nitriteMapper) { + Book entity = new Book(); + entity.bookId = nitriteMapper.convert(document.get("book_id"), BookId.class); + entity.publisher = document.get("publisher", String.class); + entity.price = document.get("price", Double.class); + entity.tags = (List) document.get("tags", List.class); + entity.description = document.get("description", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java index 239b493d3..4aa11f1af 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java @@ -19,9 +19,8 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; -import org.dizitart.no2.repository.annotations.Embedded; import static org.dizitart.no2.collection.Document.createDocument; @@ -29,26 +28,34 @@ * @author Anindya Chatterjee */ @Data -public class BookId implements Mappable { - @Embedded(order = 0) +public class BookId { private String isbn; - @Embedded(order = 1, fieldName = "book_name") private String name; private String author; - @Override - public Document write(NitriteMapper mapper) { - return createDocument("isbn", isbn) - .put("book_name", name) - .put("author", author); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - isbn = document.get("isbn", String.class); - name = document.get("book_name", String.class); - author = document.get("author", String.class); + public static class BookIdConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return BookId.class; + } + + @Override + public Document toDocument(BookId entity, NitriteMapper nitriteMapper) { + return createDocument("isbn", entity.isbn) + .put("book_name", entity.name) + .put("author", entity.author); + } + + @Override + public BookId fromDocument(Document document, NitriteMapper nitriteMapper) { + BookId entity = new BookId(); + entity.isbn = document.get("isbn", String.class); + entity.name = document.get("book_name", String.class); + entity.author = document.get("author", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java index d3c2ba644..6f0470d26 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java @@ -20,9 +20,12 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.InheritIndices; +import java.util.Date; + /** * @author Anindya Chatterjee */ @@ -32,14 +35,30 @@ public class ChildClass extends ParentClass { private String name; - @Override - public Document write(NitriteMapper mapper) { - return super.write(mapper).put("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return ChildClass.class; + } + + @Override + public Document toDocument(ChildClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("name", entity.getName()) + .put("id", entity.getId()) + .put("date", entity.getDate()) + .put("text", entity.getText()); + } - @Override - public void read(NitriteMapper mapper, Document document) { - super.read(mapper, document); - name = document.get("name", String.class); + @Override + public ChildClass fromDocument(Document document, NitriteMapper nitriteMapper) { + ChildClass entity = new ChildClass(); + entity.setId(document.get("id", Long.class)); + entity.setDate(document.get("date", Date.class)); + entity.setText(document.get("text", String.class)); + entity.setName(document.get("name", String.class)); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java index a8161c4e2..4e736981d 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java @@ -22,14 +22,14 @@ import lombok.Setter; import lombok.ToString; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.UUID; @EqualsAndHashCode @ToString -public class ClassA implements Mappable { +public class ClassA { @Getter @Setter private ClassB b; @@ -53,23 +53,33 @@ public static ClassA create(int seed) { return classA; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("b", b != null ? b.write(mapper) : null) - .put("uid", uid) - .put("string", string) - .put("blob", blob); - } + public static class ClassAConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return ClassA.class; + } + + @Override + public Document toDocument(ClassA entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("b", nitriteMapper.convert(entity.b, Document.class)) + .put("uid", entity.uid) + .put("string", entity.string) + .put("blob", entity.blob); + } - @Override - public void read(NitriteMapper mapper, Document document) { - if (document.get("b") != null) { - b = new ClassB(); - b.read(mapper, document.get("b", Document.class)); + @Override + public ClassA fromDocument(Document document, NitriteMapper nitriteMapper) { + ClassA entity = new ClassA(); + if (document.get("b") != null) { + Document doc = document.get("b", Document.class); + entity.b = nitriteMapper.convert(doc, ClassB.class); + } + entity.uid = document.get("uid", UUID.class); + entity.string = document.get("string", String.class); + entity.blob = document.get("blob", byte[].class); + return entity; } - uid = document.get("uid", UUID.class); - string = document.get("string", String.class); - blob = document.get("blob", byte[].class); } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java index a546b91e2..e77c8c5bf 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java @@ -21,16 +21,13 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; @EqualsAndHashCode @ToString -class ClassB implements Comparable, Mappable { +class ClassB implements Comparable { @Getter @Setter - private int number; + private Integer number; @Getter @Setter private String text; @@ -47,16 +44,4 @@ public int compareTo(ClassB o) { return Integer.compare(number, o.number); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("number", number) - .put("text", text); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - number = document.get("number", Integer.class); - text = document.get("text", String.class); - } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java new file mode 100644 index 000000000..6d5f88bab --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.data; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ClassBConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return ClassB.class; + } + + @Override + public Document toDocument(ClassB entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("number", entity.getNumber()) + .put("text", entity.getText()); + } + + @Override + public ClassB fromDocument(Document document, NitriteMapper nitriteMapper) { + ClassB entity = new ClassB(); + if (document.get("number") != null) { + entity.setNumber(document.get("number", Integer.class)); + } + entity.setText(document.get("text", String.class)); + return entity; + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java index 860fe1e93..0dd8d58d6 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java @@ -22,12 +22,12 @@ import lombok.Setter; import lombok.ToString; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; @EqualsAndHashCode @ToString -public class ClassC implements Mappable { +public class ClassC { @Getter @Setter private long id; @@ -46,21 +46,37 @@ public static ClassC create(int seed) { return classC; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("id", id) - .put("digit", digit) - .put("parent", parent != null ? parent.write(mapper) : null); - } + public static class ClassCConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return ClassC.class; + } + + @Override + public Document toDocument(ClassC entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("id", entity.id) + .put("digit", entity.digit) + .put("parent", nitriteMapper.convert(entity.parent, Document.class)); + } + + @Override + public ClassC fromDocument(Document document, NitriteMapper nitriteMapper) { + ClassC entity = new ClassC(); + if (document.get("id") != null) { + entity.id = document.get("id", Long.class); + } + + if (document.get("digit") != null) { + entity.digit = document.get("digit", Double.class); + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - digit = document.get("digit", Double.class); - if (document.get("parent") != null) { - parent = new ClassA(); - parent.read(mapper, document.get("parent", Document.class)); + if (document.get("parent") != null) { + Document doc = document.get("parent", Document.class); + entity.parent = nitriteMapper.convert(doc, ClassA.class); + } + return entity; } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java index 944e17358..649053261 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -37,9 +37,9 @@ */ @EqualsAndHashCode @Indices({ - @Index(value = "companyName") + @Index(fields = "companyName") }) -public class Company implements Serializable, Mappable { +public class Company implements Serializable { @Id(fieldName = "company_id") @Getter @Setter @@ -61,25 +61,6 @@ public class Company implements Serializable, Mappable { @Setter private Map> employeeRecord; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("company_id", companyId) - .put("companyName", companyName) - .put("dateCreated", dateCreated) - .put("departments", departments) - .put("employeeRecord", employeeRecord); - } - - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - companyId = document.get("company_id", Long.class); - companyName = document.get("companyName", String.class); - dateCreated = document.get("dateCreated", Date.class); - departments = document.get("departments", List.class); - employeeRecord = document.get("employeeRecord", Map.class); - } - @Override public String toString() { return "Company{" + @@ -89,4 +70,32 @@ public String toString() { ", departments=" + departments + '}'; } + + public static class CompanyConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Company.class; + } + + @Override + public Document toDocument(Company entity, NitriteMapper nitriteMapper) { + return Document.createDocument("company_id", entity.companyId) + .put("companyName", entity.companyName) + .put("dateCreated", entity.dateCreated) + .put("departments", entity.departments) + .put("employeeRecord", entity.employeeRecord); + } + + @Override + public Company fromDocument(Document document, NitriteMapper nitriteMapper) { + Company entity = new Company(); + entity.companyId = document.get("company_id", Long.class); + entity.companyName = document.get("companyName", String.class); + entity.dateCreated = document.get("dateCreated", Date.class); + entity.departments = document.get("departments", List.class); + entity.employeeRecord = document.get("employeeRecord", Map.class); + return entity; + } + } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java index 26e0dcf4a..6d83c2940 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java @@ -19,6 +19,9 @@ import com.github.javafaker.Faker; import lombok.val; +import org.dizitart.no2.integration.repository.decorator.Manufacturer; +import org.dizitart.no2.integration.repository.decorator.Product; +import org.dizitart.no2.integration.repository.decorator.ProductId; import java.nio.charset.StandardCharsets; import java.util.*; @@ -102,6 +105,15 @@ public static Book randomBook() { return book; } + public static Product randomProduct() { + Product product = new Product(); + product.setProductName(faker.name().name()); + product.setProductId(randomProductId()); + product.setManufacturer(randomManufacturer()); + product.setPrice(Double.parseDouble(faker.commerce().price())); + return product; + } + private static List departments() { return new ArrayList() {{ add("dev"); @@ -114,4 +126,19 @@ private static List departments() { add("support"); }}; } + + private static ProductId randomProductId() { + ProductId productId = new ProductId(); + productId.setProductCode(faker.code().ean13()); + productId.setUniqueId(UUID.randomUUID().toString()); + return productId; + } + + private static Manufacturer randomManufacturer() { + Manufacturer manufacturer = new Manufacturer(); + manufacturer.setUniqueId(random.nextInt()); + manufacturer.setName(faker.name().name()); + manufacturer.setAddress(faker.address().fullAddress()); + return manufacturer; + } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java index cb5304e29..92c710d2e 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.ArrayList; @@ -29,38 +29,46 @@ * @author Anindya Chatterjee */ @Data -public class ElemMatch implements Mappable { - private long id; +public class ElemMatch { + private Long id; private String[] strArray; private ProductScore[] productScores; - @Override - public Document write(NitriteMapper mapper) { - List list = new ArrayList<>(); - if (productScores != null) { - for (ProductScore productScore : productScores) { - Document document = productScore.write(mapper); - list.add(document); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return ElemMatch.class; } - return Document.createDocument("id", id) - .put("strArray", strArray) - .put("productScores", list); - } + @Override + public Document toDocument(ElemMatch entity, NitriteMapper nitriteMapper) { + List list = new ArrayList<>(); + if (entity.productScores != null) { + for (ProductScore productScore : entity.productScores) { + Document document = nitriteMapper.convert(productScore, Document.class); + list.add(document); + } + } + + return Document.createDocument("id", entity.id) + .put("strArray", entity.strArray) + .put("productScores", list); + } - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - strArray = document.get("strArray", String[].class); - List list = document.get("productScores", List.class); - if (list != null) { - productScores = new ProductScore[list.size()]; - for (int i = 0; i < list.size(); i++) { - productScores[i] = new ProductScore(); - productScores[i].read(mapper, list.get(i)); + @Override + public ElemMatch fromDocument(Document document, NitriteMapper nitriteMapper) { + ElemMatch entity = new ElemMatch(); + entity.id = document.get("id", Long.class); + entity.strArray = document.get("strArray", String[].class); + List list = document.get("productScores", List.class); + if (list != null) { + entity.productScores = new ProductScore[list.size()]; + for (int i = 0; i < list.size(); i++) { + entity.productScores[i] = nitriteMapper.convert(list.get(i), ProductScore.class); + } } + return entity; } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java index 7204f5654..f89b6ecc9 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java @@ -22,9 +22,9 @@ import lombok.Setter; import lombok.ToString; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -36,10 +36,10 @@ */ @ToString @EqualsAndHashCode -@Index(value = "joinDate", type = IndexType.NON_UNIQUE) -@Index(value = "address", type = IndexType.FULL_TEXT) -@Index(value = "employeeNote.text", type = IndexType.FULL_TEXT) -public class Employee implements Serializable, Mappable { +@Index(fields = "joinDate", type = IndexType.NON_UNIQUE) +@Index(fields = "address", type = IndexType.FULL_TEXT) +@Index(fields = "employeeNote.text", type = IndexType.FULL_TEXT) +public class Employee implements Serializable { @Id @Getter @Setter @@ -82,28 +82,39 @@ public Employee(Employee copy) { emailAddress = copy.emailAddress; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("empId", empId) - .put("joinDate", joinDate) - .put("address", address) - .put("blob", blob) - .put("emailAddress", emailAddress) - .put("employeeNote", employeeNote != null ? employeeNote.write(mapper) : null); - } + public static class EmployeeConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Employee.class; + } + + @Override + public Document toDocument(Employee entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("empId", entity.empId) + .put("joinDate", entity.joinDate) + .put("address", entity.address) + .put("blob", entity.blob) + .put("emailAddress", entity.emailAddress) + .put("employeeNote", nitriteMapper.convert(entity.employeeNote, Document.class)); + } + + @Override + public Employee fromDocument(Document document, NitriteMapper nitriteMapper) { + Employee entity = new Employee(); + + entity.empId = document.get("empId", Long.class); + entity.joinDate = document.get("joinDate", Date.class); + entity.address = document.get("address", String.class); + entity.blob = document.get("blob", byte[].class); + entity.emailAddress = document.get("emailAddress", String.class); - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - joinDate = document.get("joinDate", Date.class); - address = document.get("address", String.class); - blob = document.get("blob", byte[].class); - emailAddress = document.get("emailAddress", String.class); - - if (document.get("employeeNote") != null) { - employeeNote = new Note(); - employeeNote.read(mapper, document.get("employeeNote", Document.class)); + if (document.get("employeeNote") != null) { + Document doc = document.get("employeeNote", Document.class); + entity.employeeNote = nitriteMapper.convert(doc, Note.class);; + } + return entity; } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java new file mode 100644 index 000000000..9bf6bcbf6 --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.data; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class EmptyClass { + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return EmptyClass.class; + } + + @Override + public Document toDocument(EmptyClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument(); + } + + @Override + public EmptyClass fromDocument(Document document, NitriteMapper nitriteMapper) { + return new EmptyClass(); + } + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java index 8b9d4b23b..7003db015 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Entity; @@ -30,25 +30,35 @@ */ @Data @Entity -public class EncryptedPerson implements Mappable { +public class EncryptedPerson { private String name; private String creditCardNumber; private String cvv; private Date expiryDate; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("creditCardNumber", creditCardNumber) - .put("cvv", cvv) - .put("expiryDate", expiryDate); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return EncryptedPerson.class; + } + + @Override + public Document toDocument(EncryptedPerson entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("creditCardNumber", entity.creditCardNumber) + .put("cvv", entity.cvv) + .put("expiryDate", entity.expiryDate); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - creditCardNumber = document.get("creditCardNumber", String.class); - cvv = document.get("cvv", String.class); - expiryDate = document.get("expiryDate", Date.class); + @Override + public EncryptedPerson fromDocument(Document document, NitriteMapper nitriteMapper) { + EncryptedPerson entity = new EncryptedPerson(); + entity.name = document.get("name", String.class); + entity.creditCardNumber = document.get("creditCardNumber", String.class); + entity.cvv = document.get("cvv", String.class); + entity.expiryDate = document.get("expiryDate", Date.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java index 67dc307d1..7bf7f1726 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.io.Serializable; @@ -30,7 +30,7 @@ * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class Note implements Serializable, Mappable { +public class Note implements Serializable { @Getter @Setter private Long noteId; @@ -38,14 +38,26 @@ public class Note implements Serializable, Mappable { @Setter private String text; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument().put("noteId", noteId).put("text", text); - } + public static class NoteConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Note.class; + } + + @Override + public Document toDocument(Note entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("noteId", entity.noteId) + .put("text", entity.text); + } - @Override - public void read(NitriteMapper mapper, Document document) { - noteId = document.get("noteId", Long.class); - text = document.get("text", String.class); + @Override + public Note fromDocument(Document document, NitriteMapper nitriteMapper) { + Note entity = new Note(); + entity.noteId = document.get("noteId", Long.class); + entity.text = document.get("text", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java index bc2514543..f27a3f869 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java @@ -19,8 +19,6 @@ import lombok.Getter; import lombok.Setter; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -31,23 +29,9 @@ */ @Getter @Setter -@Index(value = "date") +@Index(fields = "date") public class ParentClass extends SuperDuperClass { @Id protected Long id; private Date date; - - @Override - public Document write(NitriteMapper mapper) { - return super.write(mapper) - .put("id", id) - .put("date", date); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - super.read(mapper, document); - id = document.get("id", Long.class); - date = document.get("date", Date.class); - } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java index 2b90075fa..b6dcd03a3 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java @@ -19,9 +19,9 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Entity; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -34,10 +34,10 @@ */ @Data @Entity(value = "MyPerson", indices = { - @Index(value = "name", type = IndexType.FULL_TEXT), - @Index(value = "status", type = IndexType.NON_UNIQUE) + @Index(fields = "name", type = IndexType.FULL_TEXT), + @Index(fields = "status", type = IndexType.NON_UNIQUE) }) -public class PersonEntity implements Mappable { +public class PersonEntity { @Id private String uuid; private String name; @@ -56,24 +56,34 @@ public PersonEntity(String name) { this.dateCreated = new Date(); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("uuid", uuid) - .put("name", name) - .put("status", status) - .put("friend", friend != null ? friend.write(mapper) : null) - .put("dateCreated", dateCreated); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return PersonEntity.class; + } + + @Override + public Document toDocument(PersonEntity entity, NitriteMapper nitriteMapper) { + return Document.createDocument("uuid", entity.uuid) + .put("name", entity.name) + .put("status", entity.status) + .put("friend", entity.friend != null ? nitriteMapper.convert(entity.friend, Document.class) : null) + .put("dateCreated", entity.dateCreated); + } - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - uuid = document.get("uuid", String.class); - name = document.get("name", String.class); - status = document.get("status", String.class); - dateCreated = document.get("dateCreated", Date.class); - friend = new PersonEntity(); - friend.read(mapper, document.get("friend", Document.class)); + @Override + public PersonEntity fromDocument(Document document, NitriteMapper nitriteMapper) { + if (document != null) { + PersonEntity entity = new PersonEntity(); + entity.uuid = document.get("uuid", String.class); + entity.name = document.get("name", String.class); + entity.status = document.get("status", String.class); + entity.dateCreated = document.get("dateCreated", Date.class); + entity.friend = nitriteMapper.convert(document.get("friend", Document.class), PersonEntity.class); + return entity; + } + return null; } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java index 65550d182..23c9d16ff 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** @@ -28,9 +28,9 @@ */ @Getter @Setter -public class ProductScore implements Mappable { +public class ProductScore { private String product; - private int score; + private Integer score; public ProductScore() { } @@ -40,15 +40,25 @@ public ProductScore(String product, int score) { this.score = score; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("product", product) - .put("score", score); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return ProductScore.class; + } + + @Override + public Document toDocument(ProductScore entity, NitriteMapper nitriteMapper) { + return Document.createDocument("product", entity.product) + .put("score", entity.score); + } - @Override - public void read(NitriteMapper mapper, Document document) { - product = document.get("product", String.class); - score = document.get("score", Integer.class); + @Override + public ProductScore fromDocument(Document document, NitriteMapper nitriteMapper) { + ProductScore entity = new ProductScore(); + entity.product = document.get("product", String.class); + entity.score = document.get("score", Integer.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java index 1ecadb243..5d9208b86 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java @@ -19,34 +19,44 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Index; /** * @author Anindya Chatterjee */ @Data -@Index(value = "firstName") -@Index(value = "age", type = IndexType.NON_UNIQUE) -@Index(value = "lastName", type = IndexType.FULL_TEXT) -public class RepeatableIndexTest implements Mappable { +@Index(fields = "firstName") +@Index(fields = "age", type = IndexType.NON_UNIQUE) +@Index(fields = "lastName", type = IndexType.FULL_TEXT) +public class RepeatableIndexTest { private String firstName; private Integer age; private String lastName; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("firstName", firstName) - .put("age", age) - .put("lastName", lastName); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return RepeatableIndexTest.class; + } + + @Override + public Document toDocument(RepeatableIndexTest entity, NitriteMapper nitriteMapper) { + return Document.createDocument("firstName", entity.firstName) + .put("age", entity.age) + .put("lastName", entity.lastName); + } - @Override - public void read(NitriteMapper mapper, Document document) { - firstName = document.get("firstName", String.class); - age = document.get("age", Integer.class); - lastName = document.get("lastName", String.class); + @Override + public RepeatableIndexTest fromDocument(Document document, NitriteMapper nitriteMapper) { + RepeatableIndexTest entity = new RepeatableIndexTest(); + entity.firstName = document.get("firstName", String.class); + entity.age = document.get("age", Integer.class); + entity.lastName = document.get("lastName", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java index d1cb199c4..33dc06073 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** @@ -28,28 +28,37 @@ */ @Getter @Setter -public class StressRecord implements Mappable { +public class StressRecord { private String firstName; - private boolean processed; + private Boolean processed; private String lastName; - private boolean failed; + private Boolean failed; private String notes; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument().put("firstName", firstName) - .put("processed", processed) - .put("lastName", lastName) - .put("failed", failed) - .put("notes", notes); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return StressRecord.class; + } + + @Override + public Document toDocument(StressRecord entity, NitriteMapper nitriteMapper) { + return Document.createDocument().put("firstName", entity.firstName) + .put("processed", entity.processed) + .put("lastName", entity.lastName) + .put("failed", entity.failed) + .put("notes", entity.notes); + } - @Override - public void read(NitriteMapper mapper, Document document) { - firstName = document.get("firstName", String.class); - processed = document.get("processed", Boolean.class); - lastName = document.get("lastName", String.class); - failed = document.get("failed", Boolean.class); - notes = document.get("notes", String.class); + @Override + public StressRecord fromDocument(Document document, NitriteMapper nitriteMapper) { + StressRecord entity = new StressRecord(); + entity.firstName = document.get("firstName", String.class); + entity.processed = document.get("processed", Boolean.class); + entity.lastName = document.get("lastName", String.class); + entity.failed = document.get("failed", Boolean.class); + entity.notes = document.get("notes", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java index 8eaa3faf1..ddea32812 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.Date; @@ -30,7 +30,7 @@ * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class SubEmployee implements Mappable { +public class SubEmployee { @Getter @Setter private Long empId; @@ -43,18 +43,28 @@ public class SubEmployee implements Mappable { @Setter private String address; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("empId", empId) - .put("joinDate", joinDate) - .put("address", address); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return SubEmployee.class; + } + + @Override + public Document toDocument(SubEmployee entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("empId", entity.empId) + .put("joinDate", entity.joinDate) + .put("address", entity.address); + } - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - joinDate = document.get("joinDate", Date.class); - address = document.get("address", String.class); + @Override + public SubEmployee fromDocument(Document document, NitriteMapper nitriteMapper) { + SubEmployee entity = new SubEmployee(); + entity.empId = document.get("empId", Long.class); + entity.joinDate = document.get("joinDate", Date.class); + entity.address = document.get("address", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java index 198f16d0c..523cebb87 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java @@ -19,10 +19,7 @@ import lombok.Getter; import lombok.Setter; -import org.dizitart.no2.collection.Document; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Index; /** @@ -30,17 +27,7 @@ */ @Getter @Setter -@Index(value = "text", type = IndexType.FULL_TEXT) -public class SuperDuperClass implements Mappable { +@Index(fields = "text", type = IndexType.FULL_TEXT) +public class SuperDuperClass { private String text; - - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("text", text); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - text = document.get("text", String.class); - } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java index 79cafe0dc..bdeef8744 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,20 +29,29 @@ */ @Getter @Setter -public class WithClassField implements Mappable { +public class WithClassField { @Id private String name; private Class clazz; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("clazz", clazz); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return WithClassField.class; + } + + @Override + public Document toDocument(WithClassField entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("clazz", entity.clazz); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - clazz = document.get("clazz", Class.class); + @Override + public WithClassField fromDocument(Document document, NitriteMapper nitriteMapper) { + WithClassField entity = new WithClassField(); + entity.name = document.get("name", String.class); + entity.clazz = document.get("clazz", Class.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java index 229f1aaf7..125e94b05 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.Date; @@ -32,19 +32,29 @@ @Getter @Setter @EqualsAndHashCode -public class WithDateId implements Mappable { +public class WithDateId { private Date id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("id", id); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithDateId.class; + } + + @Override + public Document toDocument(WithDateId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("id", entity.id); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - id = document.get("id", Date.class); + @Override + public WithDateId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithDateId entity = new WithDateId(); + entity.name = document.get("name", String.class); + entity.id = document.get("id", Date.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java index a0048a6bb..15792f12e 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,17 +29,27 @@ */ @Getter @Setter -public class WithEmptyStringId implements Mappable { +public class WithEmptyStringId { @Id private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithEmptyStringId.class; + } + + @Override + public Document toDocument(WithEmptyStringId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); + @Override + public WithEmptyStringId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithEmptyStringId entity = new WithEmptyStringId(); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java index 2364be76d..6748256c6 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java @@ -20,7 +20,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -28,21 +28,31 @@ * @author Anindya Chatterjee */ @Data -public class WithNitriteId implements Mappable { +public class WithNitriteId { @Id public NitriteId idField; public String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("idField", idField) - .put("name", name); - } + public static class WithNitriteIdConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithNitriteId.class; + } + + @Override + public Document toDocument(WithNitriteId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("idField", entity.idField) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - idField = document.get("idField", NitriteId.class); - name = document.get("name", String.class); + @Override + public WithNitriteId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithNitriteId entity = new WithNitriteId(); + entity.idField = document.get("idField", NitriteId.class); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java index 0dc3fb68b..dfac35bfa 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,21 +29,31 @@ */ @Getter @Setter -public class WithNullId implements Mappable { +public class WithNullId { @Id private String name; - private long number; + private Long number; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithNullId.class; + } + + @Override + public Document toDocument(WithNullId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithNullId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithNullId entity = new WithNullId(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java index 5f00f97d4..634a90fb8 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,17 +29,27 @@ */ @Getter @Setter -public class WithObjectId implements Mappable { +public class WithObjectId { @Id private WithOutId withOutId; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("withOutId", withOutId); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithObjectId.class; + } + + @Override + public Document toDocument(WithObjectId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("withOutId", entity.withOutId); + } - @Override - public void read(NitriteMapper mapper, Document document) { - withOutId = document.get("withOutId", WithOutId.class); + @Override + public WithObjectId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithObjectId entity = new WithObjectId(); + entity.withOutId = document.get("withOutId", WithOutId.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java index a7a2c56c9..e6783a89e 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java @@ -19,32 +19,41 @@ import lombok.EqualsAndHashCode; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class WithOutGetterSetter implements Mappable { +public class WithOutGetterSetter { private String name; - private long number; + private Long number; public WithOutGetterSetter() { name = "test"; - number = 2; + number = 2L; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("number", number); + public static class Converter implements EntityConverter { - } + @Override + public Class getEntityType() { + return WithOutGetterSetter.class; + } + + @Override + public Document toDocument(WithOutGetterSetter entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithOutGetterSetter fromDocument(Document document, NitriteMapper nitriteMapper) { + WithOutGetterSetter entity = new WithOutGetterSetter(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java index 09b63c2c2..4685e7726 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** @@ -28,25 +28,35 @@ */ @Getter @Setter -public class WithOutId implements Comparable, Mappable { +public class WithOutId implements Comparable { private String name; - private long number; + private Long number; @Override public int compareTo(WithOutId o) { return Long.compare(number, o.number); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public Class getEntityType() { + return WithOutId.class; + } + + @Override + public Document toDocument(WithOutId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("name", entity.name) + .put("number", entity.number); + } + + @Override + public WithOutId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithOutId entity = new WithOutId(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java index a425209fb..de08cf19b 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java @@ -19,38 +19,47 @@ import lombok.EqualsAndHashCode; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class WithPrivateConstructor implements Mappable { +public class WithPrivateConstructor { private String name; - private long number; + private Long number; private WithPrivateConstructor() { name = "test"; - number = 2; + number = 2L; } - public static WithPrivateConstructor create(final String name, final long number) { + public static WithPrivateConstructor create(final String name, final Long number) { WithPrivateConstructor obj = new WithPrivateConstructor(); obj.number = number; obj.name = name; return obj; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithPrivateConstructor.class; + } + + @Override + public Document toDocument(WithPrivateConstructor entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithPrivateConstructor fromDocument(Document document, NitriteMapper nitriteMapper) { + String name = document.get("name", String.class); + Long number = document.get("number", Long.class); + return WithPrivateConstructor.create(name, number); + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java index 793388231..81b69a0c1 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java @@ -18,27 +18,37 @@ package org.dizitart.no2.integration.repository.data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; /** * @author Anindya Chatterjee. */ -public class WithPublicField implements Mappable { +public class WithPublicField { @Id public String name; - public long number; + public Long number; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithPublicField.class; + } + + @Override + public Document toDocument(WithPublicField entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithPublicField fromDocument(Document document, NitriteMapper nitriteMapper) { + WithPublicField entity = new WithPublicField(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java index 9c1ade657..76ea86243 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,19 +29,29 @@ */ @Getter @Setter -public class WithTransientField implements Mappable { +public class WithTransientField { private transient String name; @Id - private long number; + private Long number; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithTransientField.class; + } + + @Override + public Document toDocument(WithTransientField entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - number = document.get("number", Long.class); + @Override + public WithTransientField fromDocument(Document document, NitriteMapper nitriteMapper) { + WithTransientField entity = new WithTransientField(); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java index 9ecb9a68b..9c721c80c 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -27,39 +27,60 @@ * @author Anindya Chatterjee */ @Data -public class WithoutEmbeddedId implements Mappable { +public class WithoutEmbeddedId { @Id private NestedId nestedId; private String data; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("nestedId", nestedId.write(mapper)) - .put("data", data); - } + @Data + public static class NestedId { + private Long id; + + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - Document nestedId = document.get("nestedId", Document.class); - this.nestedId = mapper.convert(nestedId, NestedId.class); - this.data = document.get("data", String.class); + @Override + public Class getEntityType() { + return NestedId.class; + } + + @Override + public Document toDocument(NestedId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("id", entity.id); + } + + @Override + public NestedId fromDocument(Document document, NitriteMapper nitriteMapper) { + NestedId entity = new NestedId(); + entity.id = document.get("id", Long.class); + return entity; + } + } } + public static class Converter implements EntityConverter { - @Data - public static class NestedId implements Mappable { - private Long id; + @Override + public Class getEntityType() { + return WithoutEmbeddedId.class; + } @Override - public Document write(NitriteMapper mapper) { + public Document toDocument(WithoutEmbeddedId entity, NitriteMapper nitriteMapper) { return Document.createDocument() - .put("id", id); + .put("nestedId", nitriteMapper.convert(entity.nestedId, Document.class)) + .put("data", entity.data); } @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); + public WithoutEmbeddedId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithoutEmbeddedId entity = new WithoutEmbeddedId(); + Document nestedId = document.get("nestedId", Document.class); + + entity.nestedId = nitriteMapper.convert(nestedId, NestedId.class); + entity.data = document.get("data", String.class); + + return entity; } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java new file mode 100644 index 000000000..c6bfe8bb9 --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; + +@Data +public class Manufacturer { + private String name; + private String address; + private Integer uniqueId; +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java new file mode 100644 index 000000000..f31d7b0bb --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ManufacturerConverter implements EntityConverter { + @Override + public Class getEntityType() { + return Manufacturer.class; + } + + @Override + public Document toDocument(Manufacturer entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.getName()) + .put("address", entity.getAddress()) + .put("uniqueId", entity.getUniqueId()); + } + + @Override + public Manufacturer fromDocument(Document document, NitriteMapper nitriteMapper) { + Manufacturer manufacturer = new Manufacturer(); + manufacturer.setName(document.get("name", String.class)); + manufacturer.setAddress(document.get("address", String.class)); + manufacturer.setUniqueId(document.get("uniqueId", Integer.class)); + return manufacturer; + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java new file mode 100644 index 000000000..8d066491a --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.repository.EntityIndex; +import org.dizitart.no2.repository.EntityDecorator; +import org.dizitart.no2.repository.EntityId; + +import java.util.List; + +public class ManufacturerDecorator implements EntityDecorator { + @Override + public Class getEntityType() { + return Manufacturer.class; + } + + @Override + public EntityId getIdField() { + return null; + } + + @Override + public List getIndexFields() { + return null; + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java new file mode 100644 index 000000000..89b332c8a --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +@Data +public class MiniProduct { + private String uniqueId; + private String manufacturerName; + private Double price; + + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return MiniProduct.class; + } + + @Override + public Document toDocument(MiniProduct entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("productId.uniqueId", entity.getUniqueId()) + .put("manufacturer.name", entity.getManufacturerName()) + .put("price", entity.getPrice()); + } + + @Override + public MiniProduct fromDocument(Document document, NitriteMapper nitriteMapper) { + MiniProduct entity = new MiniProduct(); + entity.setUniqueId(document.get("productId.uniqueId", String.class)); + entity.setManufacturerName(document.get("manufacturer.name", String.class)); + entity.setPrice(document.get("price", Double.class)); + return entity; + } + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java new file mode 100644 index 000000000..ff59219f9 --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; + +@Data +public class Product { + private ProductId productId; + private Manufacturer manufacturer; + private String productName; + private Double price; +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java new file mode 100644 index 000000000..0421c0cac --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ProductConverter implements EntityConverter { + @Override + public Class getEntityType() { + return Product.class; + } + + @Override + public Document toDocument(Product entity, NitriteMapper nitriteMapper) { + Document productId = nitriteMapper.convert(entity.getProductId(), Document.class); + Document manufacturer = nitriteMapper.convert(entity.getManufacturer(), Document.class); + + return Document.createDocument() + .put("productId", productId) + .put("manufacturer", manufacturer) + .put("productName", entity.getProductName()) + .put("price", entity.getPrice()); + } + + @Override + public Product fromDocument(Document document, NitriteMapper nitriteMapper) { + Product entity = new Product(); + ProductId productId = nitriteMapper.convert(document.get("productId", Document.class), ProductId.class); + Manufacturer manufacturer = nitriteMapper.convert(document.get("manufacturer", Document.class), + Manufacturer.class); + entity.setProductId(productId); + entity.setManufacturer(manufacturer); + entity.setProductName(document.get("productName", String.class)); + entity.setPrice(document.get("price", Double.class)); + return entity; + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java new file mode 100644 index 000000000..180296343 --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.repository.EntityIndex; +import org.dizitart.no2.index.IndexType; +import org.dizitart.no2.repository.EntityDecorator; +import org.dizitart.no2.repository.EntityId; + +import java.util.Arrays; +import java.util.List; + +public class ProductDecorator implements EntityDecorator { + @Override + public Class getEntityType() { + return Product.class; + } + + @Override + public EntityId getIdField() { + return new EntityId("productId", "uniqueId", "productCode"); + } + + @Override + public List getIndexFields() { + return Arrays.asList( + new EntityIndex(IndexType.NON_UNIQUE, "manufacturer.name"), + new EntityIndex(IndexType.UNIQUE, "productName", "manufacturer.uniqueId") + ); + } + + @Override + public String getEntityName() { + return "product"; + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java new file mode 100644 index 000000000..dd2f9f04e --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; + +@Data +public class ProductId { + private String uniqueId; + private String productCode; +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java new file mode 100644 index 000000000..2c02fddc0 --- /dev/null +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ProductIdConverter implements EntityConverter { + @Override + public Class getEntityType() { + return ProductId.class; + } + + @Override + public Document toDocument(ProductId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("uniqueId", entity.getUniqueId()) + .put("productCode", entity.getProductCode()); + } + + @Override + public ProductId fromDocument(Document document, NitriteMapper nitriteMapper) { + ProductId entity = new ProductId(); + entity.setUniqueId(document.get("uniqueId", String.class)); + entity.setProductCode(document.get("productCode", String.class)); + return entity; + } +} diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java index 05a2b095b..931f70a14 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java @@ -21,7 +21,7 @@ import org.dizitart.no2.integration.collection.BaseCollectionTest; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.TransactionException; import org.dizitart.no2.index.IndexType; @@ -299,11 +299,11 @@ public void testRollbackClear() { txCol.insert(document2); collection.insert(document2); - throw new TransactionException("failed"); + transaction.commit(); } catch (TransactionException e) { assert transaction != null; transaction.rollback(); - assertEquals(2, collection.size()); + assertEquals(0, collection.size()); } } } @@ -431,7 +431,7 @@ public void testCommitDropCollection() { boolean expectedException = false; try { assertEquals(0, txCol.size()); - } catch (NitriteIOException e) { + } catch (TransactionException e) { expectedException = true; } assertTrue(expectedException); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java index 5e93303ed..f1107695f 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java @@ -22,7 +22,7 @@ import org.dizitart.no2.integration.repository.data.SubEmployee; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.TransactionException; import org.dizitart.no2.index.IndexType; @@ -324,11 +324,10 @@ public void testRollbackClear() { repository.insert(txData2); transaction.commit(); - fail(); } catch (TransactionException e) { assert transaction != null; transaction.rollback(); - assertEquals(2, repository.size()); + assertEquals(0, repository.size()); } } } @@ -454,7 +453,7 @@ public void testCommitDropRepository() { boolean expectedException = false; try { assertEquals(0, txRepo.size()); - } catch (NitriteIOException e) { + } catch (TransactionException e) { expectedException = true; } assertTrue(expectedException); diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java index 1e5ed7ca6..104ccd4ad 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java @@ -21,7 +21,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -31,20 +31,30 @@ @Data @NoArgsConstructor @AllArgsConstructor -class TxData implements Mappable { +public class TxData { @Id private Long id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return TxData.class; + } + + @Override + public Document toDocument(TxData entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - name = document.get("name", String.class); + @Override + public TxData fromDocument(Document document, NitriteMapper nitriteMapper) { + TxData entity = new TxData(); + entity.id = document.get("id", Long.class); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/MVStoreModuleBuilderTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/MVStoreModuleBuilderTest.java index 0f4d3087d..ff388a2ad 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/MVStoreModuleBuilderTest.java +++ b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/MVStoreModuleBuilderTest.java @@ -78,11 +78,12 @@ public void testConstructor2() { @Test public void testFilePath() { + String tempPath = System.getProperty("java.io.tmpdir"); MVStoreModuleBuilder withConfigResult = MVStoreModule.withConfig(); MVStoreModuleBuilder actualFilePathResult = withConfigResult - .filePath(Paths.get(System.getProperty("java.io.tmpdir"), "test.txt").toFile()); + .filePath(Paths.get(tempPath, "test.txt").toFile()); assertSame(withConfigResult, actualFilePathResult); - assertEquals("/tmp/test.txt", actualFilePathResult.filePath()); + assertEquals(Paths.get(tempPath, "test.txt").toString(), actualFilePathResult.filePath()); } @Test diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/compat/v3/NitriteDataTypeTest.java b/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/compat/v3/NitriteDataTypeTest.java deleted file mode 100644 index 35b31ccff..000000000 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/compat/v3/NitriteDataTypeTest.java +++ /dev/null @@ -1,754 +0,0 @@ -/* - * Copyright (c) 2017-2021 Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.dizitart.no2.mvstore.compat.v3; - -import org.h2.mvstore.WriteBuffer; -import org.junit.Test; - -import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.UUID; - -import static org.junit.Assert.*; - -public class NitriteDataTypeTest { - @Test - public void testAutoDetectDataTypeGetType() { - assertTrue((new NitriteDataType.BigDecimalType(new NitriteDataType())) - .getType("42") instanceof NitriteDataType.StringType); - assertEquals(4, - (new NitriteDataType.BigDecimalType(new NitriteDataType())).getType(0).typeId); - } - - @Test - public void testBigDecimalTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.BigDecimalType actualBigDecimalType = new NitriteDataType.BigDecimalType(nitriteDataType); - assertEquals(9, actualBigDecimalType.typeId); - assertSame(actualBigDecimalType.base, nitriteDataType); - } - - @Test - public void testBigIntegerTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.BigIntegerType actualBigIntegerType = new NitriteDataType.BigIntegerType(nitriteDataType); - assertEquals(6, actualBigIntegerType.typeId); - assertSame(actualBigIntegerType.base, nitriteDataType); - } - - @Test - public void testBooleanTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.BooleanType actualBooleanType = new NitriteDataType.BooleanType(nitriteDataType); - assertEquals(1, actualBooleanType.typeId); - assertSame(actualBooleanType.base, nitriteDataType); - } - - @Test - public void testByteTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.ByteType actualByteType = new NitriteDataType.ByteType(nitriteDataType); - assertEquals(2, actualByteType.typeId); - assertSame(actualByteType.base, nitriteDataType); - } - - @Test - public void testCharacterTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.CharacterType actualCharacterType = new NitriteDataType.CharacterType(nitriteDataType); - assertEquals(10, actualCharacterType.typeId); - assertSame(actualCharacterType.base, nitriteDataType); - } - - @Test - public void testDateTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.DateType actualDateType = new NitriteDataType.DateType(nitriteDataType); - assertEquals(13, actualDateType.typeId); - assertSame(actualDateType.base, nitriteDataType); - } - - @Test - public void testDoubleTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.DoubleType actualDoubleType = new NitriteDataType.DoubleType(nitriteDataType); - assertEquals(8, actualDoubleType.typeId); - assertSame(actualDoubleType.base, nitriteDataType); - } - - @Test - public void testFloatTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.FloatType actualFloatType = new NitriteDataType.FloatType(nitriteDataType); - assertEquals(7, actualFloatType.typeId); - assertSame(actualFloatType.base, nitriteDataType); - } - - @Test - public void testIntegerTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.IntegerType actualIntegerType = new NitriteDataType.IntegerType(nitriteDataType); - assertEquals(4, actualIntegerType.typeId); - assertSame(actualIntegerType.base, nitriteDataType); - } - - @Test - public void testIsBigInteger() { - assertFalse(NitriteDataType.isBigInteger("Obj")); - assertFalse(NitriteDataType.isBigInteger(null)); - } - - @Test - public void testIsBigDecimal() { - assertFalse(NitriteDataType.isBigDecimal("Obj")); - assertFalse(NitriteDataType.isBigDecimal(null)); - } - - @Test - public void testIsDate() { - assertFalse(NitriteDataType.isDate("Obj")); - assertFalse(NitriteDataType.isDate(null)); - } - - @Test - public void testIsArray() { - assertFalse(NitriteDataType.isArray("Obj")); - assertFalse(NitriteDataType.isArray(null)); - } - - @Test - public void testGetCommonClassId() { - assertEquals(8, NitriteDataType.getCommonClassId(Object.class).intValue()); - assertNull(NitriteDataType.getCommonClassId(null)); - } - - @Test - public void testLongTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.LongType actualLongType = new NitriteDataType.LongType(nitriteDataType); - assertEquals(5, actualLongType.typeId); - assertSame(actualLongType.base, nitriteDataType); - } - - @Test - public void testNullTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.NullType actualNullType = new NitriteDataType.NullType(nitriteDataType); - assertEquals(0, actualNullType.typeId); - assertSame(actualNullType.base, nitriteDataType); - } - - @Test - public void testObjectArrayTypeConstructor() { - assertEquals(14, (new NitriteDataType.ObjectArrayType(new NitriteDataType())).typeId); - } - - @Test - public void testSerialize() { - byte[] actualSerializeResult = NitriteDataType.serialize("Obj"); - assertEquals(10, actualSerializeResult.length); - assertEquals((byte) 0, actualSerializeResult[2]); - assertEquals((byte) 5, actualSerializeResult[3]); - assertEquals('t', actualSerializeResult[4]); - assertEquals((byte) 0, actualSerializeResult[5]); - assertEquals((byte) 3, actualSerializeResult[6]); - assertEquals('O', actualSerializeResult[7]); - assertEquals('b', actualSerializeResult[8]); - assertEquals('j', actualSerializeResult[9]); - } - - @Test - public void testCompareNotNull() throws UnsupportedEncodingException { - byte[] data1 = "AAAAAAAA".getBytes("UTF-8"); - assertEquals(0, NitriteDataType.compareNotNull(data1, "AAAAAAAA".getBytes("UTF-8"))); - } - - @Test - public void testCompareNotNull2() throws UnsupportedEncodingException { - assertEquals(-1, - NitriteDataType.compareNotNull(new byte[]{0, 'A', 'A', 'A', 'A', 'A', 'A', 'A'}, "AAAAAAAA".getBytes("UTF-8"))); - } - - @Test - public void testCompareNotNull3() throws UnsupportedEncodingException { - byte[] data1 = "ï¿¿AAAAAAA".getBytes("UTF-8"); - assertEquals(1, NitriteDataType.compareNotNull(data1, "AAAAAAAA".getBytes("UTF-8"))); - } - - @Test - public void testAutoDetectDataTypeCompare() { - assertEquals(-1, (new NitriteDataType.BigDecimalType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.BigDecimalType(new NitriteDataType())).compare(0, "B Obj")); - } - - @Test - public void testBigDecimalTypeCompare() { - assertEquals(-1, (new NitriteDataType.BigDecimalType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.BigDecimalType(new NitriteDataType())).compare(0, "B Obj")); - } - - @Test - public void testBigIntegerTypeCompare() { - assertEquals(-1, (new NitriteDataType.BigIntegerType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.BigIntegerType(new NitriteDataType())).compare(0, "B Obj")); - } - - @Test - public void testBooleanTypeCompare() { - assertEquals(-1, (new NitriteDataType.BooleanType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.BooleanType(new NitriteDataType())).compare(true, "B Obj")); - assertEquals(0, (new NitriteDataType.BooleanType(new NitriteDataType())).compare(true, true)); - } - - @Test - public void testByteTypeCompare() { - assertEquals(-1, (new NitriteDataType.ByteType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.ByteType(new NitriteDataType())).compare((byte) 'A', "B Obj")); - assertEquals(0, (new NitriteDataType.ByteType(new NitriteDataType())).compare((byte) 'A', (byte) 'A')); - } - - @Test - public void testCharacterTypeCompare() { - assertEquals(-1, (new NitriteDataType.CharacterType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.CharacterType(new NitriteDataType())).compare('\u0000', "B Obj")); - assertEquals(0, (new NitriteDataType.CharacterType(new NitriteDataType())).compare('\u0000', '\u0000')); - } - - @Test - public void testCompare() { - assertEquals(0, (new NitriteDataType()).compare("42", "42")); - assertEquals(-1, (new NitriteDataType()).compare(0, "42")); - assertEquals(1, (new NitriteDataType()).compare("42", 0)); - assertEquals(0, (new NitriteDataType()).compare(0, 0)); - } - - @Test - public void testDateTypeCompare() { - assertEquals(-1, (new NitriteDataType.DateType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.DateType(new NitriteDataType())).compare(0, "B Obj")); - } - - @Test - public void testDoubleTypeCompare() { - assertEquals(-1, (new NitriteDataType.DoubleType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.DoubleType(new NitriteDataType())).compare(0, "B Obj")); - assertEquals(-1, (new NitriteDataType.DoubleType(new NitriteDataType())).compare(10.0, "B Obj")); - assertEquals(0, (new NitriteDataType.DoubleType(new NitriteDataType())).compare(10.0, 10.0)); - } - - @Test - public void testFloatTypeCompare() { - assertEquals(-1, (new NitriteDataType.FloatType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.FloatType(new NitriteDataType())).compare(0, "B Obj")); - assertEquals(-1, (new NitriteDataType.FloatType(new NitriteDataType())).compare(10.0f, "B Obj")); - assertEquals(0, (new NitriteDataType.FloatType(new NitriteDataType())).compare(10.0f, 10.0f)); - } - - @Test - public void testIntegerTypeCompare() { - assertEquals(-1, (new NitriteDataType.IntegerType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.IntegerType(new NitriteDataType())).compare(0, "B Obj")); - assertEquals(0, (new NitriteDataType.IntegerType(new NitriteDataType())).compare(0, 0)); - } - - @Test - public void testLongTypeCompare() { - assertEquals(-1, (new NitriteDataType.LongType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.LongType(new NitriteDataType())).compare(0L, "B Obj")); - assertEquals(0, (new NitriteDataType.LongType(new NitriteDataType())).compare(0L, 0L)); - } - - @Test - public void testNullTypeCompare() { - assertEquals(-1, (new NitriteDataType.NullType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.NullType(new NitriteDataType())).compare(0, "B Obj")); - assertEquals(-1, (new NitriteDataType.NullType(new NitriteDataType())).compare(null, "B Obj")); - assertEquals(1, (new NitriteDataType.NullType(new NitriteDataType())).compare("A Obj", null)); - assertEquals(0, (new NitriteDataType.NullType(new NitriteDataType())).compare(null, null)); - } - - @Test - public void testObjectArrayTypeCompare() { - assertEquals(-1, (new NitriteDataType.ObjectArrayType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.ObjectArrayType(new NitriteDataType())).compare(0, "B Obj")); - } - - @Test - public void testSerializedObjectTypeCompare() { - assertEquals(-1, (new NitriteDataType.SerializedObjectType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.SerializedObjectType(new NitriteDataType())).compare(0, "B Obj")); - assertEquals(0, (new NitriteDataType.SerializedObjectType(new NitriteDataType())).compare(0, 0)); - } - - @Test - public void testShortTypeCompare() { - assertEquals(-1, (new NitriteDataType.ShortType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.ShortType(new NitriteDataType())).compare((short) 0, "B Obj")); - assertEquals(0, (new NitriteDataType.ShortType(new NitriteDataType())).compare((short) 0, (short) 0)); - } - - @Test - public void testStringTypeCompare() { - assertEquals(-1, (new NitriteDataType.StringType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.StringType(new NitriteDataType())).compare(0, "B Obj")); - assertEquals(1, (new NitriteDataType.StringType(new NitriteDataType())).compare("A Obj", 0)); - assertEquals(-1, (new NitriteDataType.StringType(new NitriteDataType())).compare(4, "B Obj")); - assertEquals(0, (new NitriteDataType.StringType(new NitriteDataType())).compare(0, 0)); - } - - @Test - public void testUUIDTypeCompare() { - assertEquals(-1, (new NitriteDataType.UUIDType(new NitriteDataType())).compare("A Obj", "B Obj")); - assertEquals(-1, (new NitriteDataType.UUIDType(new NitriteDataType())).compare(0, "B Obj")); - } - - @Test - public void testUUIDTypeCompare2() { - NitriteDataType.UUIDType uuidType = new NitriteDataType.UUIDType(new NitriteDataType()); - assertEquals(1, uuidType.compare(UUID.randomUUID(), "B Obj")); - } - - @Test - public void testAutoDetectDataTypeGetMemory() { - assertEquals(28, (new NitriteDataType.BigDecimalType(new NitriteDataType())).getMemory("42")); - assertEquals(24, (new NitriteDataType.BigDecimalType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testBigDecimalTypeGetMemory() { - assertEquals(30, (new NitriteDataType.BigDecimalType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.BigDecimalType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testBigDecimalTypeGetMemory2() { - NitriteDataType nitriteDataType = new NitriteDataType(); - nitriteDataType.switchType(1); - assertEquals(30, (new NitriteDataType.BigDecimalType(nitriteDataType)).getMemory("Obj")); - } - - @Test - public void testBigIntegerTypeGetMemory() { - assertEquals(30, (new NitriteDataType.BigIntegerType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.BigIntegerType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testBigIntegerTypeGetMemory2() { - NitriteDataType nitriteDataType = new NitriteDataType(); - nitriteDataType.switchType(1); - assertEquals(30, (new NitriteDataType.BigIntegerType(nitriteDataType)).getMemory("Obj")); - } - - @Test - public void testBooleanTypeGetMemory() { - assertEquals(30, (new NitriteDataType.BooleanType(new NitriteDataType())).getMemory("Obj")); - assertEquals(0, (new NitriteDataType.BooleanType(new NitriteDataType())).getMemory(true)); - assertEquals(24, (new NitriteDataType.BooleanType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testByteTypeGetMemory() { - assertEquals(30, (new NitriteDataType.ByteType(new NitriteDataType())).getMemory("Obj")); - assertEquals(0, (new NitriteDataType.ByteType(new NitriteDataType())).getMemory((byte) 'A')); - assertEquals(24, (new NitriteDataType.ByteType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testCharacterTypeGetMemory() { - assertEquals(30, (new NitriteDataType.CharacterType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.CharacterType(new NitriteDataType())).getMemory('\u0000')); - assertEquals(24, (new NitriteDataType.CharacterType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testDateTypeGetMemory() { - assertEquals(30, (new NitriteDataType.DateType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.DateType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testDateTypeGetMemory2() { - NitriteDataType nitriteDataType = new NitriteDataType(); - nitriteDataType.switchType(1); - assertEquals(30, (new NitriteDataType.DateType(nitriteDataType)).getMemory("Obj")); - } - - @Test - public void testDoubleTypeGetMemory() { - assertEquals(30, (new NitriteDataType.DoubleType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.DoubleType(new NitriteDataType())).getMemory(0)); - assertEquals(30, (new NitriteDataType.DoubleType(new NitriteDataType())).getMemory(10.0)); - } - - @Test - public void testFloatTypeGetMemory() { - assertEquals(30, (new NitriteDataType.FloatType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.FloatType(new NitriteDataType())).getMemory(0)); - assertEquals(24, (new NitriteDataType.FloatType(new NitriteDataType())).getMemory(10.0f)); - } - - @Test - public void testGetMemory() { - assertEquals(30, (new NitriteDataType()).getMemory("Obj")); - assertEquals(24, (new NitriteDataType()).getMemory(0)); - } - - @Test - public void testGetMemory2() { - NitriteDataType nitriteDataType = new NitriteDataType(); - nitriteDataType.switchType(1); - assertEquals(30, nitriteDataType.getMemory("Obj")); - } - - @Test - public void testIntegerTypeGetMemory() { - assertEquals(30, (new NitriteDataType.IntegerType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.IntegerType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testIntegerTypeGetMemory2() { - NitriteDataType nitriteDataType = new NitriteDataType(); - nitriteDataType.switchType(1); - assertEquals(30, (new NitriteDataType.IntegerType(nitriteDataType)).getMemory("Obj")); - } - - @Test - public void testLongTypeGetMemory() { - assertEquals(30, (new NitriteDataType.LongType(new NitriteDataType())).getMemory("Obj")); - assertEquals(30, (new NitriteDataType.LongType(new NitriteDataType())).getMemory(0L)); - assertEquals(24, (new NitriteDataType.LongType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testNullTypeGetMemory() { - assertEquals(30, (new NitriteDataType.NullType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.NullType(new NitriteDataType())).getMemory(0)); - assertEquals(0, (new NitriteDataType.NullType(new NitriteDataType())).getMemory(null)); - } - - @Test - public void testObjectArrayTypeGetMemory() { - assertEquals(30, (new NitriteDataType.ObjectArrayType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.ObjectArrayType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testObjectArrayTypeGetMemory2() { - NitriteDataType nitriteDataType = new NitriteDataType(); - nitriteDataType.switchType(1); - assertEquals(30, (new NitriteDataType.ObjectArrayType(nitriteDataType)).getMemory("Obj")); - } - - @Test - public void testSerializedObjectTypeGetMemory() { - assertEquals(30, (new NitriteDataType.SerializedObjectType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.SerializedObjectType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testSerializedObjectTypeGetMemory2() { - NitriteDataType.SerializedObjectType serializedObjectType = new NitriteDataType.SerializedObjectType( - new NitriteDataType()); - serializedObjectType.write(new WriteBuffer(), 19088743); - assertEquals(30, serializedObjectType.getMemory("Obj")); - } - - @Test - public void testShortTypeGetMemory() { - assertEquals(30, (new NitriteDataType.ShortType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.ShortType(new NitriteDataType())).getMemory((short) 0)); - assertEquals(24, (new NitriteDataType.ShortType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testStringTypeGetMemory() { - assertEquals(30, (new NitriteDataType.StringType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.StringType(new NitriteDataType())).getMemory(0)); - assertEquals(24, (new NitriteDataType.StringType(new NitriteDataType())).getMemory(4)); - } - - @Test - public void testStringTypeGetMemory2() { - NitriteDataType nitriteDataType = new NitriteDataType(); - nitriteDataType.switchType(1); - assertEquals(24, (new NitriteDataType.StringType(nitriteDataType)).getMemory(0)); - } - - @Test - public void testUUIDTypeGetMemory() { - assertEquals(30, (new NitriteDataType.UUIDType(new NitriteDataType())).getMemory("Obj")); - assertEquals(24, (new NitriteDataType.UUIDType(new NitriteDataType())).getMemory(0)); - } - - @Test - public void testUUIDTypeGetMemory2() { - NitriteDataType.UUIDType uuidType = new NitriteDataType.UUIDType(new NitriteDataType()); - assertEquals(40, uuidType.getMemory(UUID.randomUUID())); - } - - @Test - public void testBigDecimalTypeRead() { - Object actualReadResult = (new NitriteDataType.BigDecimalType(new NitriteDataType())).read(null, 46); - assertSame(((BigDecimal) actualReadResult).ZERO, actualReadResult); - } - - @Test - public void testBigDecimalTypeRead2() { - Object actualReadResult = (new NitriteDataType.BigDecimalType(new NitriteDataType())).read(null, 47); - assertSame(((BigDecimal) actualReadResult).ONE, actualReadResult); - } - - @Test - public void testBigIntegerTypeRead() { - Object actualReadResult = (new NitriteDataType.BigIntegerType(new NitriteDataType())).read(null, 37); - assertSame(((BigInteger) actualReadResult).ZERO, actualReadResult); - } - - @Test - public void testBigIntegerTypeRead2() { - Object actualReadResult = (new NitriteDataType.BigIntegerType(new NitriteDataType())).read(null, 38); - assertSame(((BigInteger) actualReadResult).ONE, actualReadResult); - } - - @Test - public void testNullTypeRead() { - assertNull((new NitriteDataType.NullType(new NitriteDataType())).read(null, 1)); - } - - @Test - public void testStringTypeRead2() { - assertEquals("", (new NitriteDataType.StringType(new NitriteDataType())).read(null, 88)); - } - - @Test - public void testAutoDetectDataTypeWrite() { - NitriteDataType.BigDecimalType bigDecimalType = new NitriteDataType.BigDecimalType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - bigDecimalType.write(writeBuffer, "42"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testAutoDetectDataTypeWrite2() { - NitriteDataType.BigDecimalType bigDecimalType = new NitriteDataType.BigDecimalType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - bigDecimalType.write(writeBuffer, new Object[]{"42", "42", "42"}, 3, true); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testAutoDetectDataTypeWrite3() { - NitriteDataType.BigDecimalType bigDecimalType = new NitriteDataType.BigDecimalType(null); - assertThrows(ArrayIndexOutOfBoundsException.class, - () -> bigDecimalType.write(new WriteBuffer(), new Object[]{}, 3, true)); - } - - @Test - public void testBigDecimalTypeWrite() { - NitriteDataType.BigDecimalType bigDecimalType = new NitriteDataType.BigDecimalType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - bigDecimalType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testBigIntegerTypeWrite() { - NitriteDataType.BigIntegerType bigIntegerType = new NitriteDataType.BigIntegerType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - bigIntegerType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testBooleanTypeWrite() { - NitriteDataType.BooleanType booleanType = new NitriteDataType.BooleanType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - booleanType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testByteTypeWrite() { - NitriteDataType.ByteType byteType = new NitriteDataType.ByteType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - byteType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testCharacterTypeWrite() { - NitriteDataType.CharacterType characterType = new NitriteDataType.CharacterType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - characterType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testDateTypeWrite() { - NitriteDataType.DateType dateType = new NitriteDataType.DateType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - dateType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testDoubleTypeWrite() { - NitriteDataType.DoubleType doubleType = new NitriteDataType.DoubleType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - doubleType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testFloatTypeWrite() { - NitriteDataType.FloatType floatType = new NitriteDataType.FloatType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - floatType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testIntegerTypeWrite() { - NitriteDataType.IntegerType integerType = new NitriteDataType.IntegerType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - integerType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testLongTypeWrite() { - NitriteDataType.LongType longType = new NitriteDataType.LongType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - longType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testNullTypeWrite() { - NitriteDataType.NullType nullType = new NitriteDataType.NullType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - nullType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testObjectArrayTypeWrite() { - NitriteDataType.ObjectArrayType objectArrayType = new NitriteDataType.ObjectArrayType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - objectArrayType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testSerializedObjectTypeWrite() { - NitriteDataType.SerializedObjectType serializedObjectType = new NitriteDataType.SerializedObjectType( - new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - serializedObjectType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testShortTypeWrite() { - NitriteDataType.ShortType shortType = new NitriteDataType.ShortType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - shortType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testStringTypeWrite() { - NitriteDataType.StringType stringType = new NitriteDataType.StringType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - stringType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testUUIDTypeWrite() { - NitriteDataType.UUIDType uuidType = new NitriteDataType.UUIDType(new NitriteDataType()); - WriteBuffer writeBuffer = new WriteBuffer(); - uuidType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testSerializedObjectTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.SerializedObjectType actualSerializedObjectType = new NitriteDataType.SerializedObjectType( - nitriteDataType); - assertEquals(19, actualSerializedObjectType.typeId); - assertSame(actualSerializedObjectType.base, nitriteDataType); - } - - @Test - public void testShortTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.ShortType actualShortType = new NitriteDataType.ShortType(nitriteDataType); - assertEquals(3, actualShortType.typeId); - assertSame(actualShortType.base, nitriteDataType); - } - - @Test - public void testStringTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.StringType actualStringType = new NitriteDataType.StringType(nitriteDataType); - assertEquals(11, actualStringType.typeId); - assertSame(actualStringType.base, nitriteDataType); - } - - @Test - public void testUUIDTypeConstructor() { - NitriteDataType nitriteDataType = new NitriteDataType(); - NitriteDataType.UUIDType actualUuidType = new NitriteDataType.UUIDType(nitriteDataType); - assertEquals(12, actualUuidType.typeId); - assertSame(actualUuidType.base, nitriteDataType); - } - - @Test - public void testWrite() { - NitriteDataType nitriteDataType = new NitriteDataType(); - WriteBuffer writeBuffer = new WriteBuffer(); - nitriteDataType.write(writeBuffer, "Obj"); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testWrite4() { - NitriteDataType nitriteDataType = new NitriteDataType(); - WriteBuffer writeBuffer = new WriteBuffer(); - nitriteDataType.write(writeBuffer, new Object[]{"42", "42", "42"}, 3, true); - assertEquals(1048576, writeBuffer.capacity()); - } - - @Test - public void testWrite5() { - assertThrows(ArrayIndexOutOfBoundsException.class, - () -> (new NitriteDataType()).write(null, new Object[]{}, 3, true)); - } - - @Test - public void testSwitchType() { - assertTrue((new NitriteDataType()).switchType("Obj") instanceof NitriteDataType.StringType); - assertEquals(4, (new NitriteDataType()).switchType(0).typeId); - } -} - diff --git a/nitrite-replication/build.gradle b/nitrite-replication/build.gradle index 8d8f09ef3..ab06cf8d8 100644 --- a/nitrite-replication/build.gradle +++ b/nitrite-replication/build.gradle @@ -48,17 +48,29 @@ dependencies { api "org.slf4j:slf4j-api" api "com.fasterxml.jackson.core:jackson-databind" api "com.squareup.okhttp3:okhttp" - annotationProcessor "org.projectlombok:lombok:1.18.22" + annotationProcessor "org.projectlombok:lombok:1.18.24" testImplementation project(path: ':nitrite-mvstore-adapter', configuration: 'default') - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" + testAnnotationProcessor "org.projectlombok:lombok:1.18.24" testImplementation "jakarta.websocket:jakarta.websocket-api:2.0.0" - testImplementation "org.glassfish.tyrus:tyrus-server:2.0.1" - testImplementation "org.glassfish.tyrus:tyrus-container-grizzly-server:2.0.1" - testImplementation "org.awaitility:awaitility:4.1.1" - testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.14.1" - testImplementation "org.apache.logging.log4j:log4j-core:2.15.0" + testImplementation 'org.glassfish.tyrus:tyrus-server:2.0.2' + testImplementation 'org.glassfish.tyrus:tyrus-container-grizzly-server:2.0.2' + testImplementation 'org.awaitility:awaitility:4.2.0' + testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.2' + testImplementation 'org.apache.logging.log4j:log4j-core:2.17.2' testImplementation "junit:junit:4.13.2" + testImplementation "org.testcontainers:testcontainers:1.17.2" + testImplementation 'org.testcontainers:mongodb:1.17.2' + testImplementation 'org.mongodb:mongo-java-driver:3.12.11' + testImplementation 'commons-lang:commons-lang:2.6' + testImplementation ("com.github.javafaker:javafaker:1.0.2") { + exclude module: 'snakeyaml' + } + testImplementation ("com.palantir.docker.compose:docker-compose-rule-junit4:1.7.0") { + exclude module: 'guava' + } + testImplementation 'org.yaml:snakeyaml:1.30' + testImplementation 'com.google.guava:guava:31.1-jre' } test { diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/BatchChangeScheduler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/BatchChangeScheduler.java deleted file mode 100644 index 2beb1adf9..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/BatchChangeScheduler.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync; - -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.dizitart.no2.sync.message.BatchChangeContinue; -import org.dizitart.no2.sync.message.BatchChangeEnd; -import org.dizitart.no2.sync.message.BatchChangeStart; - -import java.util.Timer; -import java.util.TimerTask; - -/** - * @author Anindya Chatterjee - */ -class BatchChangeScheduler { - private Timer timer; - private final ReplicationTemplate replica; - private final MessageFactory factory; - private final MessageTemplate messageTemplate; - private final FeedJournal journal; - - public BatchChangeScheduler(ReplicationTemplate replica) { - this.replica = replica; - this.factory = replica.getMessageFactory(); - this.messageTemplate = replica.getMessageTemplate(); - this.journal = replica.getFeedJournal(); - } - - public void schedule() { - if (replica.isConnected()) { - Long lastSyncTime = replica.getLastSyncTime(); - - BatchChangeStart message = createStart(factory, lastSyncTime); - messageTemplate.sendMessage(message); - journal.write(message.getFeed()); - - timer = new Timer(); - timer.scheduleAtFixedRate(new TimerTask() { - boolean hasMore = true; - int start = replica.getConfig().getChunkSize(); - - @Override - public void run() { - LastWriteWinState state = replica.getCrdt().getChangesSince(lastSyncTime, start, - replica.getConfig().getChunkSize()); - if (state.getChanges().size() == 0 && state.getTombstones().size() == 0) { - hasMore = false; - } - - if (hasMore) { - BatchChangeContinue message = factory.createChangeContinue(replica.getConfig(), - replica.getReplicaId(), "", state); - - messageTemplate.sendMessage(message); - journal.write(state); - start = start + replica.getConfig().getChunkSize(); - } - - if (!hasMore) { - timer.cancel(); - } - } - }, 0, replica.getConfig().getDebounce()); - - BatchChangeEnd endMessage = factory.createChangeEnd(replica.getConfig(), replica.getReplicaId(), "", lastSyncTime); - messageTemplate.sendMessage(endMessage); - } - } - - public void stop() { - if (timer != null) { - timer.cancel(); - } - } - - private BatchChangeStart createStart(MessageFactory factory, Long lastSyncTime) { - BatchChangeStart startMessage = factory.createChangeStart(replica.getConfig(), - replica.getReplicaId(), ""); - - LastWriteWinState state = replica.getCrdt().getChangesSince(lastSyncTime, 0, - replica.getConfig().getChunkSize()); - - startMessage.setFeed(state); - return startMessage; - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/BatchChangeSender.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/BatchChangeSender.java new file mode 100644 index 000000000..b64ede68f --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/BatchChangeSender.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2017-2020. Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dizitart.no2.sync; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.crdt.ConflictFreeReplicatedDataType; +import org.dizitart.no2.sync.crdt.DeltaStates; +import org.dizitart.no2.sync.crdt.Markers; +import org.dizitart.no2.sync.message.*; + +/** + * @author Anindya Chatterjee + */ +@Slf4j +public class BatchChangeSender { + private final Config config; + private final ReplicatedCollection replicatedCollection; + private final String replicaId; + private final ConflictFreeReplicatedDataType replicatedDataType; + private final DataGateSocketListener dataGateSocketListener; + private final FeedLedger feedLedger; + + private boolean hasMore; + private State currentState; + private MessageFactory messageFactory; + private Markers startMarkers; + private Markers endMarkers; + + public BatchChangeSender(Config config, + ReplicatedCollection replicatedCollection, + DataGateSocketListener dataGateSocketListener) { + this.config = config; + this.replicatedCollection = replicatedCollection; + this.replicaId = replicatedCollection.getReplicaId(); + this.feedLedger = replicatedCollection.getFeedLedger(); + this.replicatedDataType = replicatedCollection.getReplicatedDataType(); + this.dataGateSocketListener = dataGateSocketListener; + configure(); + } + + public void sendAndReceive(WebSocket webSocket, BatchMessage batchMessage) { + if (startMarkers == null) { + startMarkers = replicatedDataType.getLocalStartMarkers(); + } + + if (endMarkers == null) { + endMarkers = replicatedDataType.getLocalEndMarkers(); + } + + switch (currentState) { + case ReadyToSend: + sendStartMessage(webSocket, batchMessage); + break; + case StartSent: + sendChanges(webSocket, batchMessage); + break; + case ChangesSent: + break; + } + } + + private void sendStartMessage(WebSocket webSocket, BatchMessage batchMessage) { + BatchChangeStart startMessage = messageFactory.createChangeStart(config, + replicaId, batchMessage.getHeader().getTransactionId()); + startMessage.setNextOffset(config.getChunkSize()); + startMessage.setBatchSize(config.getChunkSize()); + + DeltaStates state = replicatedDataType.delta(startMarkers, endMarkers, + 0, config.getChunkSize()); + + startMessage.setFeed(state); + dataGateSocketListener.sendMessage(webSocket, startMessage); + feedLedger.writeEntry(startMessage.getFeed()); + + currentState = State.StartSent; + } + + private void sendChanges(WebSocket webSocket, BatchMessage batchMessage) { + DeltaStates state = replicatedDataType.delta(startMarkers, endMarkers, + batchMessage.getNextOffset(), config.getChunkSize()); + + if (state.getChangeSet().size() == 0 && state.getTombstoneMap().size() == 0) { + hasMore = false; + } + + if (hasMore) { + BatchChangeContinue message = messageFactory.createChangeContinue(config, + replicaId, batchMessage.getHeader().getTransactionId(), state); + message.setNextOffset(batchMessage.getNextOffset() + config.getChunkSize()); + message.setBatchSize(config.getChunkSize()); + + dataGateSocketListener.sendMessage(webSocket, message); + feedLedger.writeEntry(state); + } else { + Receipt finalReceipt = feedLedger.getFinalReceipt(); + if (replicatedCollection.shouldRetry(finalReceipt)) { + state = replicatedCollection.createState(finalReceipt); + BatchChangeContinue message = messageFactory.createChangeContinue(config, + replicaId, batchMessage.getHeader().getTransactionId(), state); + + dataGateSocketListener.sendMessage(webSocket, message); + feedLedger.writeEntry(state); + } else { + sendEndMessage(webSocket, batchMessage.getHeader().getTransactionId()); + currentState = State.ChangesSent; + } + } + } + + private void sendEndMessage(WebSocket webSocket, String correlationId) { + BatchChangeEnd endMessage = messageFactory.createChangeEnd(config, + replicatedCollection.getReplicaId(), correlationId); + + Markers serverStartMarkers = replicatedDataType.getRemoteStartMarkers(); + endMessage.setBatchSize(config.getChunkSize()); + endMessage.setStartMarkers(serverStartMarkers); + + // end markers will be passed back in batch end ack message + endMessage.setEndMarkers(endMarkers); + + dataGateSocketListener.sendMessage(webSocket, endMessage); + } + + private void configure() { + this.messageFactory = new MessageFactory(); + this.hasMore = true; + this.currentState = State.ReadyToSend; + } + + private enum State { + ReadyToSend, + StartSent, + ChangesSent + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/Config.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/Config.java index 89eba4e56..76bfccf53 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/Config.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/Config.java @@ -19,25 +19,33 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; import okhttp3.Request; +import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.sync.event.ReplicationEventListener; import java.net.Proxy; -import java.util.concurrent.Callable; +import java.util.List; /** + * Represents the replication configuration + * * @author Anindya Chatterjee + * @since 4.0.0 */ @Data public class Config { + private Nitrite db; private NitriteCollection collection; private Integer chunkSize; private String userName; - private Integer debounce; + private String tenant; + private Integer pollingRate; private ObjectMapper objectMapper; private TimeSpan timeout; private Request.Builder requestBuilder; private Proxy proxy; private String authToken; private boolean acceptAllCertificates; - private Callable networkConnectivityChecker; + private List eventListeners; + private String replicaName; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/DataGateSocketListener.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/DataGateSocketListener.java new file mode 100644 index 000000000..ca58f00ff --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/DataGateSocketListener.java @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.sync; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import org.dizitart.no2.common.util.StringUtils; +import org.dizitart.no2.sync.event.ReplicationEvent; +import org.dizitart.no2.sync.event.ReplicationEventBus; +import org.dizitart.no2.sync.event.ReplicationEventListener; +import org.dizitart.no2.sync.event.ReplicationEventType; +import org.dizitart.no2.sync.message.Connect; +import org.dizitart.no2.sync.message.DataGateMessage; +import org.dizitart.no2.sync.message.Disconnect; +import org.dizitart.no2.sync.net.CloseReason; +import org.dizitart.no2.sync.net.WebSocketCode; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.UUID; + +import static org.dizitart.no2.sync.event.ReplicationEventType.Started; +import static org.dizitart.no2.sync.event.ReplicationEventType.Stopped; + +/** + * @author Anindya Chatterjee + */ +@Slf4j +public class DataGateSocketListener extends WebSocketListener { + private final Config config; + private final ReplicatedCollection replicatedCollection; + + private ReplicationEventBus eventBus; + private MessageFactory messageFactory; + private MessageTemplate messageTemplate; + private MessageTransformer transformer; + private WebSocket connectedWebsocket; + + public DataGateSocketListener(Config config, ReplicatedCollection replicatedCollection) { + this.config = config; + this.replicatedCollection = replicatedCollection; + configure(); + } + + @Override + public void onOpen(@NotNull WebSocket webSocket, @NotNull Response response) { + try { + if (!StringUtils.isNullOrEmpty(config.getReplicaName())) { + Thread.currentThread().setName(config.getReplicaName()); + } + + log.debug("Connection opened, sending Connect message"); + this.connectedWebsocket = webSocket; + String correlationId = UUID.randomUUID().toString(); + Connect message = messageFactory.createConnect(config, replicatedCollection.getReplicaId(), correlationId); + messageTemplate.postMessage(webSocket, message); + eventBus.post(new ReplicationEvent(Started)); + } catch (Exception e) { + log.error("Opening websocket failed", e); + eventBus.post(new ReplicationEvent(ReplicationEventType.Error, e)); + closeConnection(webSocket, new CloseReason("Client Error", e)); + } + } + + @Override + public void onMessage(@NotNull WebSocket webSocket, @NotNull String text) { + try { + log.debug("Received message from server {}", text); + DataGateMessage message = transformer.transform(text); + messageTemplate.validateMessage(message); + messageTemplate.dispatchMessage(webSocket, message); + } catch (Exception e) { + log.error("Error while processing message", e); + eventBus.post(new ReplicationEvent(ReplicationEventType.Error, e)); + + if (e instanceof ReplicationException) { + if (((ReplicationException) e).isFatal()) { + closeConnection(webSocket, new CloseReason("Client Error", e)); + } + } + } + } + + @Override + public void onFailure(@NotNull WebSocket webSocket, @NotNull Throwable t, @Nullable Response response) { + log.error("Communication failure", t); + eventBus.post(new ReplicationEvent(ReplicationEventType.Error, t)); + closeConnection(webSocket, new CloseReason("Client Error", t)); + } + + @Override + public void onClosed(@NotNull WebSocket webSocket, int code, @NotNull String reason) { + log.warn("Connection to server is closed due to {}", reason); + eventBus.post(new ReplicationEvent(Stopped)); + replicatedCollection.setStopped(true); + } + + public void sendMessage(WebSocket webSocket, M message) { + try { + if (!replicatedCollection.isStopped()) { + messageTemplate.postMessage(webSocket, message); + } else { + throw new IllegalStateException("Datagate client is not connected"); + } + } catch (Exception e) { + log.error("Failed to send message", e); + eventBus.post(new ReplicationEvent(ReplicationEventType.Error, e)); + closeConnection(webSocket, new CloseReason("Client Error", e)); + } + } + + public void closeConnection(WebSocket webSocket, CloseReason reason) { + log.debug("Closing connection due to {}", reason); + + if (reason == CloseReason.ClientClose) { + Disconnect disconnect = messageFactory.createDisconnect(config, + replicatedCollection.getReplicaId(), UUID.randomUUID().toString()); + if (connectedWebsocket != null) { + messageTemplate.postMessage(connectedWebsocket, disconnect); + } + } + + replicatedCollection.setStopped(true); + + if (webSocket != null) { + webSocket.close(WebSocketCode.NORMAL_CLOSE, reason.getReasonMessage()); + } else { + if (connectedWebsocket != null) { + connectedWebsocket.close(WebSocketCode.NORMAL_CLOSE, reason.getReasonMessage()); + } + } + + eventBus.post(new ReplicationEvent(Stopped)); + } + + private void configure() { + eventBus = new ReplicationEventBus(); + messageFactory = new MessageFactory(); + messageTemplate = new MessageTemplate(config, replicatedCollection); + transformer = new MessageTransformer(config.getObjectMapper()); + + if (config.getEventListeners() != null) { + for (ReplicationEventListener eventListener : config.getEventListeners()) { + eventBus.register(eventListener); + } + } + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/FeedJournal.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/FeedJournal.java deleted file mode 100644 index a14e6715b..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/FeedJournal.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.collection.meta.Attributes; -import org.dizitart.no2.common.util.StringUtils; -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.dizitart.no2.sync.message.Receipt; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.locks.ReentrantLock; - -/** - * @author Anindya Chatterjee - */ -@Slf4j -public class FeedJournal { - private static final String JOURNAL = "journal"; - - private final ReplicationTemplate replicationTemplate; - private final ReentrantLock lock; - - public FeedJournal(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; - this.lock = new ReentrantLock(); - } - - public Receipt accumulate(Receipt receipt) { - try { - lock.lock(); - Receipt current = getCurrent(); - if (receipt != null && current != null) { - for (String id : receipt.getAdded()) { - current.getAdded().remove(id); - } - - for (String id : receipt.getRemoved()) { - current.getRemoved().remove(id); - } - } - setCurrent(current); - return current; - } finally { - lock.unlock(); - } - } - - public void write(LastWriteWinState state) { - try { - lock.lock(); - if (state != null) { - Receipt receipt = getCurrent(); - - Set changes = state.getChanges(); - if (changes != null && !changes.isEmpty()) { - for (Document change : changes) { - receipt.getAdded().add(change.getId().getIdValue()); - } - } - - Map tombstones = state.getTombstones(); - if (tombstones != null && !tombstones.isEmpty()) { - for (Map.Entry entry : tombstones.entrySet()) { - receipt.getRemoved().add(entry.getKey()); - } - } - - setCurrent(receipt); - } - } finally { - lock.unlock(); - } - } - - public Receipt getFinalReceipt() { - try { - lock.lock(); - return getCurrent(); - } finally { - lock.unlock(); - } - } - - private Receipt getCurrent() { - try { - Attributes attributes = replicationTemplate.getAttributes(); - String json = attributes.get(JOURNAL); - if (StringUtils.isNullOrEmpty(json)) { - return new Receipt(new HashSet<>(), new HashSet<>()); - } - - ObjectMapper objectMapper = replicationTemplate.getConfig().getObjectMapper(); - return objectMapper.readValue(json, Receipt.class); - } catch (JsonProcessingException e) { - log.error("Error while opening replica ledger", e); - throw new ReplicationException("failed to open replica ledger", e, false); - } - } - - private void setCurrent(Receipt receipt) { - try { - ObjectMapper objectMapper = replicationTemplate.getConfig().getObjectMapper(); - String json = objectMapper.writeValueAsString(receipt); - Attributes attributes = replicationTemplate.getAttributes(); - attributes.set(JOURNAL, json); - - replicationTemplate.saveAttributes(attributes); - } catch (JsonProcessingException e) { - log.error("Error while writing replica ledger", e); - throw new ReplicationException("failed to write replica ledger", e, false); - } - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/FeedLedger.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/FeedLedger.java new file mode 100644 index 000000000..e68ce7571 --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/FeedLedger.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2017-2020. Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dizitart.no2.sync; + +import lombok.extern.slf4j.Slf4j; +import org.dizitart.no2.Nitrite; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.common.meta.Attributes; +import org.dizitart.no2.common.util.StringUtils; +import org.dizitart.no2.sync.crdt.DeltaStates; +import org.dizitart.no2.sync.message.Receipt; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.dizitart.no2.common.meta.Attributes.FEED_LEDGER; + +/** + * @author Anindya Chatterjee + */ +@Slf4j +public class FeedLedger { + + private final Config config; + private NitriteCollection journal; + + public FeedLedger(Config config) { + this.config = config; + initializeLedger(); + } + + public void writeOff(Receipt receipt) { + Receipt current = getCurrent(); + if (receipt != null) { + for (String id : receipt.getAdded()) { + current.getAdded().remove(id); + } + + for (String id : receipt.getRemoved()) { + current.getRemoved().remove(id); + } + } + setCurrent(current); + } + + public void writeEntry(DeltaStates state) { + if (state != null) { + Receipt receipt = getCurrent(); + + Set changes = state.getChangeSet(); + if (changes != null && !changes.isEmpty()) { + for (Document change : changes) { + receipt.getAdded().add(change.getId().getIdValue()); + } + } + + Map tombstones = state.getTombstoneMap(); + if (tombstones != null && !tombstones.isEmpty()) { + for (Map.Entry entry : tombstones.entrySet()) { + receipt.getRemoved().add(entry.getKey()); + } + } + + setCurrent(receipt); + } + } + + public Receipt getFinalReceipt() { + return getCurrent(); + } + + private void initializeLedger() { + NitriteCollection collection = config.getCollection(); + String feedLedgerName = getFeedLedgerName(collection); + + Nitrite db = config.getDb(); + this.journal = db.getCollection(feedLedgerName); + } + + private String getFeedLedgerName(NitriteCollection collection) { + Attributes attributes = collection.getAttributes(); + String feedLedgerName = attributes.get(FEED_LEDGER); + if (StringUtils.isNullOrEmpty(feedLedgerName)) { + feedLedgerName = collection.getName() + "_" + FEED_LEDGER; + attributes.set(FEED_LEDGER, feedLedgerName); + collection.setAttributes(attributes); + } + return feedLedgerName; + } + + private Receipt getCurrent() { + Document document = journal.find().firstOrNull(); + if (document == null) { + Receipt receipt = new Receipt(new HashSet<>(), new HashSet<>()); + document = receipt.toDocument(); + journal.insert(document); + return receipt; + } else { + return Receipt.fromDocument(document); + } + } + + private void setCurrent(Receipt receipt) { + Document document = journal.find().firstOrNull(); + if (document == null) { + document = receipt.toDocument(); + journal.insert(document); + } else { + document.put("added", receipt.getAdded()); + document.put("removed", receipt.getRemoved()); + journal.update(document); + } + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageDispatcher.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageDispatcher.java deleted file mode 100644 index a279130d4..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageDispatcher.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync; - -import lombok.extern.slf4j.Slf4j; -import okhttp3.Response; -import org.dizitart.no2.common.Constants; -import org.dizitart.no2.common.concurrent.ThreadPoolManager; -import org.dizitart.no2.common.util.StringUtils; -import org.dizitart.no2.sync.event.ReplicationEvent; -import org.dizitart.no2.sync.event.ReplicationEventType; -import org.dizitart.no2.sync.handlers.*; -import org.dizitart.no2.sync.message.DataGateMessage; -import org.dizitart.no2.sync.net.DataGateSocketListener; - -import java.util.concurrent.ExecutorService; - -/** - * @author Anindya Chatterjee - */ -@Slf4j -class MessageDispatcher implements DataGateSocketListener, AutoCloseable { - private final ReplicationTemplate replicationTemplate; - private final MessageTransformer transformer; - private ExecutorService executorService; - - public MessageDispatcher(Config config, ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; - this.transformer = new MessageTransformer(config.getObjectMapper()); - } - - @Override - public void onMessage(String text) { - try { - log.debug("Message received from server {}", text); - DataGateMessage message = transformer.transform(text); - validateMessage(message); - dispatch(message); - } catch (Exception e) { - log.error("Error while processing message", e); - replicationTemplate.postEvent(new ReplicationEvent(ReplicationEventType.Error, e)); - replicationTemplate.stopReplication("Error - " + e.getMessage()); - } - } - - @Override - public void onFailure(Throwable t, Response response) { - log.error("Communication failure", t); - replicationTemplate.postEvent(new ReplicationEvent(ReplicationEventType.Error, t)); - replicationTemplate.stopReplication("Error - " + t.getMessage()); - } - - @Override - public void onClosed(int code, String reason) { - log.warn("Connection to server is closed due to {}", reason); - } - - private void dispatch(M message) { - MessageHandler handler = findHandler(message); - if (handler != null) { - getExecutorService().submit(() -> { - try { - handler.handleMessage(message); - } catch (ReplicationException error) { - log.error("Error occurred while handling {} message", message.getHeader().getMessageType(), error); - if (error.isFatal()) { - replicationTemplate.postEvent(new ReplicationEvent(ReplicationEventType.Error, error)); - replicationTemplate.stopReplication("Error - " + error.getMessage()); - } - } catch (Exception e) { - log.error("Error occurred while handling {} message", message.getHeader().getMessageType(), e); - replicationTemplate.postEvent(new ReplicationEvent(ReplicationEventType.Error, e)); - replicationTemplate.stopReplication("Error - " + e.getMessage()); - } - }); - } - } - - @SuppressWarnings("unchecked") - private MessageHandler findHandler(DataGateMessage message) { - switch (message.getHeader().getMessageType()) { - case Error: - return (MessageHandler) new ErrorHandler(replicationTemplate); - case Connect: - // impossible case, server will never initiate connection - break; - case ConnectAck: - return (MessageHandler) new ConnectAckHandler(replicationTemplate); - case Disconnect: - return (MessageHandler) new DisconnectHandler(replicationTemplate); - case BatchChangeStart: - return (MessageHandler) new BatchChangeStartHandler(replicationTemplate); - case BatchChangeContinue: - return (MessageHandler) new BatchChangeContinueHandler(replicationTemplate); - case BatchChangeEnd: - return (MessageHandler) new BatchChangeEndHandler(replicationTemplate); - case BatchAck: - return (MessageHandler) new BatchAckHandler(replicationTemplate); - case BatchEndAck: - return (MessageHandler) new BatchEndAckHandler(replicationTemplate); - case DataGateFeed: - if (replicationTemplate.shouldExchangeFeed()) { - return (MessageHandler) new DataGateFeedHandler(replicationTemplate); - } - break; - case DataGateFeedAck: - if (replicationTemplate.shouldExchangeFeed()) { - return (MessageHandler) new DataGateFeedAckHandler(replicationTemplate); - } - break; - } - return null; - } - - private void validateMessage(DataGateMessage message) { - if (message == null) { - throw new ReplicationException("a null message is received for " - + replicationTemplate.getReplicaId(), true); - } else if (message.getHeader() == null) { - throw new ReplicationException("a message without header is received for " - + replicationTemplate.getReplicaId(), true); - } else if (StringUtils.isNullOrEmpty(message.getHeader().getCollection())) { - throw new ReplicationException("a message without collection info is received for " - + replicationTemplate.getReplicaId(), true); - } else if (message.getHeader().getMessageType() == null) { - throw new ReplicationException("a message without any type is received for " - + replicationTemplate.getReplicaId(), true); - } - } - - private ExecutorService getExecutorService() { - if (executorService == null - || executorService.isShutdown() - || executorService.isTerminated()) { - int core = Runtime.getRuntime().availableProcessors(); - executorService = ThreadPoolManager.getThreadPool(core, Constants.SYNC_THREAD_NAME); - } - return executorService; - } - - @Override - public void close() { - if (executorService != null) { - executorService.shutdown(); - } - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageFactory.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageFactory.java index a84239ef5..2180c6c5b 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageFactory.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageFactory.java @@ -16,7 +16,7 @@ package org.dizitart.no2.sync; -import org.dizitart.no2.sync.crdt.LastWriteWinState; +import org.dizitart.no2.sync.crdt.DeltaStates; import org.dizitart.no2.sync.message.*; import java.util.UUID; @@ -25,94 +25,94 @@ * @author Anindya Chatterjee */ public class MessageFactory { - public Connect createConnect(Config config, String replicaId) { + public Connect createConnect(Config config, String replicaId, String txId) { Connect message = new Connect(); message.setHeader(createHeader(MessageType.Connect, config.getCollection().getName(), - "", replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); message.setAuthToken(config.getAuthToken()); return message; } - public Disconnect createDisconnect(Config config, String replicaId) { + public Disconnect createDisconnect(Config config, String replicaId, String txId) { Disconnect message = new Disconnect(); message.setHeader(createHeader(MessageType.Disconnect, config.getCollection().getName(), - "", replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); return message; } - public BatchChangeStart createChangeStart(Config config, String replicaId, String uuid) { + public BatchChangeStart createChangeStart(Config config, String replicaId, String txId) { BatchChangeStart message = new BatchChangeStart(); message.setHeader(createHeader(MessageType.BatchChangeStart, config.getCollection().getName(), - uuid, replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); message.setBatchSize(config.getChunkSize()); - message.setDebounce(config.getDebounce()); return message; } public BatchChangeContinue createChangeContinue(Config config, String replicaId, - String uuid, LastWriteWinState state) { + String txId, DeltaStates state) { BatchChangeContinue message = new BatchChangeContinue(); message.setHeader(createHeader(MessageType.BatchChangeContinue, config.getCollection().getName(), - uuid, replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); message.setBatchSize(config.getChunkSize()); - message.setDebounce(config.getDebounce()); message.setFeed(state); return message; } - public BatchChangeEnd createChangeEnd(Config config, String replicaId, String uuid, Long lastSyncTime) { + public BatchChangeEnd createChangeEnd(Config config, String replicaId, + String txId) { BatchChangeEnd message = new BatchChangeEnd(); message.setHeader(createHeader(MessageType.BatchChangeEnd, config.getCollection().getName(), - uuid, replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); message.setBatchSize(config.getChunkSize()); - message.setDebounce(config.getDebounce()); - message.setLastSynced(lastSyncTime); return message; } - public DataGateFeed createFeedMessage(Config config, String replicaId, LastWriteWinState state) { + public DataGateFeed createFeedMessage(Config config, String replicaId, + String txId, DeltaStates state) { DataGateFeed feed = new DataGateFeed(); feed.setHeader(createHeader(MessageType.DataGateFeed, config.getCollection().getName(), - "", replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); feed.setFeed(state); return feed; } public DataGateFeedAck createFeedAck(Config config, String replicaId, - String correlationId, Receipt receipt) { + String txId, Receipt receipt) { DataGateFeedAck ack = new DataGateFeedAck(); ack.setHeader(createHeader(MessageType.DataGateFeedAck, config.getCollection().getName(), - correlationId, replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); ack.setReceipt(receipt); return ack; } public BatchAck createBatchAck(Config config, String replicaId, - String correlationId, Receipt receipt) { + String txId, Receipt receipt) { BatchAck ack = new BatchAck(); ack.setHeader(createHeader(MessageType.BatchAck, config.getCollection().getName(), - correlationId, replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); ack.setReceipt(receipt); return ack; } - public BatchEndAck createBatchEndAck(Config config, String replicaId, String correlationId) { + public BatchEndAck createBatchEndAck(Config config, String replicaId, String txId) { BatchEndAck ack = new BatchEndAck(); ack.setHeader(createHeader(MessageType.BatchEndAck, config.getCollection().getName(), - correlationId, replicaId, config.getUserName())); + txId, replicaId, config.getUserName(), config.getTenant())); return ack; } public MessageHeader createHeader(MessageType messageType, String collectionName, - String correlationId, String replicaId, String userName) { + String txId, String replicaId, + String userName, String tenant) { MessageHeader messageHeader = new MessageHeader(); messageHeader.setId(UUID.randomUUID().toString()); - messageHeader.setCorrelationId(correlationId); + messageHeader.setTransactionId(txId); messageHeader.setCollection(collectionName); messageHeader.setMessageType(messageType); messageHeader.setOrigin(replicaId); messageHeader.setTimestamp(System.currentTimeMillis()); messageHeader.setUserName(userName); + messageHeader.setTenant(tenant); return messageHeader; } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTemplate.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTemplate.java index 89e1f0062..a81c405da 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTemplate.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTemplate.java @@ -1,79 +1,101 @@ /* - * Copyright (c) 2017-2020. Nitrite author or authors. + * Copyright (c) 2017-2021 Nitrite author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ package org.dizitart.no2.sync; +import com.fasterxml.jackson.core.JsonProcessingException; import lombok.extern.slf4j.Slf4j; +import okhttp3.WebSocket; +import org.dizitart.no2.common.util.StringUtils; +import org.dizitart.no2.sync.handlers.*; import org.dizitart.no2.sync.message.DataGateMessage; -import org.dizitart.no2.sync.net.DataGateSocket; /** * @author Anindya Chatterjee */ @Slf4j -public class MessageTemplate implements AutoCloseable { +public class MessageTemplate { private final Config config; - private final ReplicationTemplate replica; - private DataGateSocket dataGateSocket; - private MessageDispatcher dispatcher; + private final ReplicatedCollection replicatedCollection; - public MessageTemplate(Config config, ReplicationTemplate replica) { + public MessageTemplate(Config config, ReplicatedCollection replicatedCollection) { this.config = config; - this.replica = replica; + this.replicatedCollection = replicatedCollection; } - public void sendMessage(DataGateMessage message) { - if (dataGateSocket != null && dataGateSocket.isConnected()) { - if (!dataGateSocket.sendMessage(message)) { - throw new ReplicationException("failed to deliver message " + message, true); - } + public void validateMessage(DataGateMessage message) { + if (message == null) { + throw new ReplicationException("A null message is received for " + + replicatedCollection.getReplicaId(), true); + } else if (message.getHeader() == null) { + throw new ReplicationException("A message without header is received for " + + replicatedCollection.getReplicaId(), true); + } else if (StringUtils.isNullOrEmpty(message.getHeader().getCollection())) { + throw new ReplicationException("A message without collection info is received for " + + replicatedCollection.getReplicaId(), true); + } else if (message.getHeader().getMessageType() == null) { + throw new ReplicationException("A message without any type is received for " + + replicatedCollection.getReplicaId(), true); } } - public void openConnection() { - try { - dataGateSocket = new DataGateSocket(config); - dispatcher = new MessageDispatcher(config, replica); - - dataGateSocket.setListener(dispatcher); - dataGateSocket.startConnect(); - } catch (Exception e) { - log.error("Error while establishing connection from {}", getReplicaId(), e); - throw new ReplicationException("failed to open connection to server", e, true); + public void dispatchMessage(WebSocket webSocket, M message) { + MessageHandler handler = findHandler(message); + if (handler != null) { + handler.handleMessage(webSocket, message); } } - public void closeConnection(String reason) { - if (dataGateSocket != null) { - dataGateSocket.stopConnect(reason); + public void postMessage(WebSocket webSocket, M message) { + try { + String text = config.getObjectMapper().writeValueAsString(message); + log.debug("Sending message to datagate server {}", text); + webSocket.send(text); + } catch (JsonProcessingException e) { + throw new ReplicationException("Malformed datagate message", e, true); } } - @Override - public void close() { - if (dataGateSocket != null) { - dataGateSocket.stopConnect("normal close"); + @SuppressWarnings("unchecked") + private MessageHandler findHandler(DataGateMessage message) { + switch (message.getHeader().getMessageType()) { + case Error: + return (MessageHandler) new ErrorHandler(); + case ConnectAck: + return (MessageHandler) new ConnectAckHandler(replicatedCollection); + case Disconnect: + return (MessageHandler) new DisconnectHandler(replicatedCollection); + case BatchChangeStart: + return (MessageHandler) new BatchChangeStartHandler(replicatedCollection); + case BatchChangeContinue: + return (MessageHandler) new BatchChangeContinueHandler(replicatedCollection); + case BatchChangeEnd: + return (MessageHandler) new BatchChangeEndHandler(replicatedCollection); + case BatchAck: + return (MessageHandler) new BatchAckHandler(replicatedCollection); + case BatchEndAck: + return (MessageHandler) new BatchEndAckHandler(replicatedCollection); + case DataGateFeed: + return (MessageHandler) new DataGateFeedHandler(replicatedCollection); + case DataGateFeedAck: + return (MessageHandler) new DataGateFeedAckHandler(replicatedCollection); + default: + break; } - - if (dispatcher != null) { - dispatcher.close(); - } - } - - private String getReplicaId() { - return replica.getReplicaId(); + return null; } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTransformer.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTransformer.java index 8e7b652f0..3f6532222 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTransformer.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/MessageTransformer.java @@ -25,7 +25,7 @@ * @author Anindya Chatterjee */ public class MessageTransformer { - private ObjectMapper objectMapper; + private final ObjectMapper objectMapper; public MessageTransformer(ObjectMapper objectMapper) { this.objectMapper = objectMapper; @@ -58,7 +58,7 @@ public DataGateMessage transform(String message) { return objectMapper.treeToValue(jsonNode, DataGateFeedAck.class); } } catch (JsonProcessingException e) { - throw new ReplicationException("failed to transform message from server", e, true); + throw new ReplicationException("Failed to transform message from server", e, true); } return null; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/Replica.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/Replica.java index d8417fdd2..b9b8a5a06 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/Replica.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/Replica.java @@ -1,35 +1,50 @@ /* - * Copyright (c) 2017-2020. Nitrite author or authors. + * Copyright (c) 2017-2021 Nitrite author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ package org.dizitart.no2.sync; -import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.sync.event.ReplicationEvent; -import org.dizitart.no2.sync.event.ReplicationEventListener; -import org.dizitart.no2.sync.event.ReplicationEventType; +import org.dizitart.no2.common.concurrent.ThreadPoolManager; +import org.dizitart.no2.sync.net.CloseReason; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.dizitart.no2.common.Constants.SYNC_THREAD_NAME; /** + * Represents a remote replica of the local {@link org.dizitart.no2.collection.NitriteCollection} + * or {@link org.dizitart.no2.repository.ObjectRepository}. + * * @author Anindya Chatterjee + * @since 4.0.0 */ -@Slf4j -public final class Replica implements AutoCloseable { - private final ReplicationTemplate replicationTemplate; +public class Replica implements AutoCloseable { + private final Config config; + private final ReplicatedCollection replicatedCollection; + private AtomicBoolean disconnected; + private ScheduledExecutorService scheduledExecutorService; + private ScheduledFuture replicationTask; - Replica(Config config) { - this.replicationTemplate = new ReplicationTemplate(config); + Replica(Config config, ReplicatedCollection replicatedCollection) { + this.config = config; + this.replicatedCollection = replicatedCollection; + configure(); } public static ReplicaBuilder builder() { @@ -37,50 +52,60 @@ public static ReplicaBuilder builder() { } public void connect() { - try { - replicationTemplate.connect(); - } catch (Exception e) { - log.error("Error while connecting the replica {}", getReplicaId(), e); - replicationTemplate.postEvent(new ReplicationEvent(ReplicationEventType.Error, e)); - if (e instanceof ReplicationException) { - throw e; + if (scheduledExecutorService != null) { + if (scheduledExecutorService.isShutdown() || scheduledExecutorService.isTerminated()) { + scheduledExecutorService = getSyncThreadPool(); } - throw new ReplicationException("failed to open connection", e, true); + + disconnected.compareAndSet(true, false); + + if (replicationTask == null || replicationTask.isCancelled()) { + replicationTask = scheduledExecutorService.scheduleAtFixedRate(() -> { + if (replicatedCollection.isStopped() && !disconnected.get()) { + replicatedCollection.startReplication(); + } + }, 0, config.getPollingRate(), TimeUnit.MILLISECONDS); + } + } else { + throw new ReplicationException("Replica is not configured properly", true); } } public void disconnect() { - try { - replicationTemplate.disconnect(); - } catch (Exception e) { - replicationTemplate.postEvent(new ReplicationEvent(ReplicationEventType.Error, e)); - log.error("Error while disconnecting the replica {}", getReplicaId(), e); - if (e instanceof ReplicationException) { - throw e; - } - throw new ReplicationException("failed to disconnect the replica", e, true); - } + disconnectInternal(false); } - public void subscribe(ReplicationEventListener listener) { - replicationTemplate.subscribe(listener); + public void disconnectNow() { + disconnectInternal(true); } - public void unsubscribe(ReplicationEventListener listener) { - replicationTemplate.unsubscribe(listener); + public boolean isDisconnected() { + return disconnected.get() || replicatedCollection.isStopped(); } - private String getReplicaId() { - return replicationTemplate.getReplicaId(); + public void close() { + disconnected.compareAndSet(false, true); + replicatedCollection.stopReplication(null, CloseReason.ClientClose); + ThreadPoolManager.shutdownThreadPool(scheduledExecutorService); } - public boolean isConnected() { - return replicationTemplate.isConnected(); + private void disconnectInternal(boolean mayInterruptIfRunning) { + if (scheduledExecutorService != null) { + if (!scheduledExecutorService.isShutdown() && !scheduledExecutorService.isTerminated()) { + replicationTask.cancel(mayInterruptIfRunning); + disconnected.compareAndSet(false, true); + } + } else { + throw new ReplicationException("Replica is not configured properly", true); + } } - @Override - public void close() { - replicationTemplate.stopReplication("Normal shutdown"); - replicationTemplate.close(); + private void configure() { + this.scheduledExecutorService = getSyncThreadPool(); + this.disconnected = new AtomicBoolean(true); + } + + private static ScheduledExecutorService getSyncThreadPool() { + return ThreadPoolManager.getScheduledThreadPool(1, SYNC_THREAD_NAME); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicaBuilder.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicaBuilder.java index 4572b1b3e..cfc59377d 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicaBuilder.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicaBuilder.java @@ -17,135 +17,291 @@ package org.dizitart.no2.sync; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; import okhttp3.Request; +import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.NitriteCollection; import org.dizitart.no2.repository.ObjectRepository; +import org.dizitart.no2.sync.event.ReplicationEventListener; import org.dizitart.no2.sync.module.DocumentModule; -import java.math.BigInteger; import java.net.Proxy; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.Callable; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; import java.util.concurrent.TimeUnit; +import static org.dizitart.no2.common.util.StringUtils.isNullOrEmpty; + /** + * A builder api for creating a nitrite {@link Replica}. + * * @author Anindya Chatterjee. + * @since 4.0.0 */ +@Slf4j public class ReplicaBuilder { + private Nitrite db; private NitriteCollection collection; - private String replicationServer; + private String remoteHost; + private Integer remotePort; private String authToken; private TimeSpan timeout; - private TimeSpan debounce; + private TimeSpan pollingRate; private Integer chunkSize; + private String tenant; private String userName; private ObjectMapper objectMapper; private Proxy proxy; private boolean acceptAllCertificates = false; - private Callable networkConnectivityChecker = () -> true; + private final List eventListeners; + private String replicaName; + /** + * Instantiates a new {@link ReplicaBuilder}. + */ ReplicaBuilder() { chunkSize = 10; + remotePort = 46005; // nitrite molar mass timeout = new TimeSpan(5, TimeUnit.SECONDS); - debounce = new TimeSpan(1, TimeUnit.SECONDS); + pollingRate = new TimeSpan(1, TimeUnit.SECONDS); objectMapper = new ObjectMapper(); objectMapper.registerModule(new DocumentModule()); + eventListeners = new ArrayList<>(); + } + + /** + * Creates a replica from a {@link Nitrite} database. + * + * @param db the db + * @return the replica builder + */ + public ReplicaBuilder database(Nitrite db) { + this.db = db; + return this; } + /** + * Creates a replica of a {@link NitriteCollection}. + * + * @param collection the collection + * @return the replica builder + */ public ReplicaBuilder of(NitriteCollection collection) { this.collection = collection; return this; } + /** + * Creates a replica of an {@link ObjectRepository}. + * + * @param repository the repository + * @return the replica builder + */ public ReplicaBuilder of(ObjectRepository repository) { return of(repository.getDocumentCollection()); } - public ReplicaBuilder remote(String replicationServer) { - this.replicationServer = replicationServer; + /** + * Sets the remote datagate server host. + * + * @param remoteHost the replication server host + * @return the replica builder + */ + public ReplicaBuilder remoteHost(String remoteHost) { + this.remoteHost = remoteHost; return this; } - public ReplicaBuilder jwtAuth(String userName, String authToken) { - this.authToken = authToken; - this.userName = userName; + /** + * Sets the remote datagate server port. + * + * @param remotePort the replication server port + * @return the replica builder + */ + public ReplicaBuilder remotePort(Integer remotePort) { + this.remotePort = remotePort; + return this; + } + + /** + * Sets the remote datagate server tenant id. + * + * @param tenantId the replication server tenant id + * @return the replica builder + */ + public ReplicaBuilder tenant(String tenantId) { + this.tenant = tenantId; return this; } - public ReplicaBuilder basicAuth(String userName, String password) { - this.authToken = toHex(userName + ":" + password); + /** + * Sets the JWT auth token and username. + * + * @param userName the username + * @param authToken the auth token + * @return the replica builder + */ + public ReplicaBuilder jwtAuth(String userName, String authToken) { + this.authToken = authToken; this.userName = userName; return this; } + /** + * Sets the connection timeout. + * + * @param timeSpan the time span + * @return the replica builder + */ public ReplicaBuilder timeout(TimeSpan timeSpan) { this.timeout = timeSpan; return this; } + /** + * Sets the chunk size of changes that will be transmitted. + * + * @param size the size + * @return the replica builder + */ public ReplicaBuilder chunkSize(Integer size) { this.chunkSize = size; return this; } - public ReplicaBuilder debounce(TimeSpan timeSpan) { - this.debounce = timeSpan; + /** + * Sets the polling rate value. + * + * @param timeSpan the time span + * @return the replica builder + */ + public ReplicaBuilder pollingRate(TimeSpan timeSpan) { + this.pollingRate = timeSpan; return this; } + /** + * Sets the {@link ObjectMapper} instance. + * + * @param objectMapper the object mapper + * @return the replica builder + */ public ReplicaBuilder objectMapper(ObjectMapper objectMapper) { this.objectMapper = objectMapper; return this; } + /** + * Sets the proxy details. + * + * @param proxy the proxy + * @return the replica builder + */ public ReplicaBuilder proxy(Proxy proxy) { this.proxy = proxy; return this; } + /** + * Sets a flag to accept all certificates. + * + * @param accept to accept + * @return the replica builder + */ public ReplicaBuilder acceptAllCertificates(boolean accept) { this.acceptAllCertificates = accept; return this; } - public ReplicaBuilder networkConnectivityChecker(Callable callable) { - this.networkConnectivityChecker = callable; + /** + * Add a replication event listener to the replica. + * + * @param listener the listener + * @return the replica builder + */ + public ReplicaBuilder addReplicationEventListener(ReplicationEventListener listener) { + this.eventListeners.add(listener); return this; } + /** + * Sets an optional name for the replica. + * + * @param name the name of the replica + * @return the replica builder + */ + public ReplicaBuilder replicaName(String name) { + this.replicaName = name; + return this; + } + + /** + * Creates a {@link Replica}. + * + * @return the replica + */ public Replica create() { - if (collection != null) { - Request.Builder builder = createRequestBuilder(); - - Config config = new Config(); - config.setCollection(collection); - config.setChunkSize(chunkSize); - config.setUserName(userName); - config.setDebounce(getTimeoutInMillis(debounce)); - config.setObjectMapper(objectMapper); - config.setTimeout(timeout); - config.setRequestBuilder(builder); - config.setProxy(proxy); - config.setAcceptAllCertificates(acceptAllCertificates); - config.setAuthToken(authToken); - config.setNetworkConnectivityChecker(networkConnectivityChecker); - return new Replica(config); - } else { - throw new ReplicationException("no collection or repository has been specified for replication", true); - } + validateBuilder(); + Request.Builder builder = createRequestBuilder(); + + Config config = new Config(); + config.setDb(db); + config.setCollection(collection); + config.setChunkSize(chunkSize); + config.setUserName(userName); + config.setTenant(tenant); + config.setPollingRate(getTimeoutInMillis(pollingRate)); + config.setObjectMapper(objectMapper); + config.setTimeout(timeout); + config.setRequestBuilder(builder); + config.setProxy(proxy); + config.setAcceptAllCertificates(acceptAllCertificates); + config.setAuthToken(authToken); + config.setEventListeners(eventListeners); + config.setReplicaName(replicaName); + + ReplicatedCollection replicatedCollection = new ReplicatedCollection(config); + return new Replica(config, replicatedCollection); } private Request.Builder createRequestBuilder() { + String remoteUrl = String.format(Locale.getDefault(), "ws://%s:%d/ws/datagate/%s/%s/%s", + remoteHost, remotePort, tenant, collection.getName(), userName); + + log.debug("Using remote datagate url " + remoteUrl); Request.Builder builder = new Request.Builder(); - builder.url(replicationServer); + builder.url(remoteUrl); return builder; } - private String toHex(String arg) { - return String.format("%040x", new BigInteger(1, arg.getBytes(StandardCharsets.UTF_8))); - } - private int getTimeoutInMillis(TimeSpan connectTimeout) { return Math.toIntExact(connectTimeout.getTimeUnit().toMillis(connectTimeout.getTime())); } + + private void validateBuilder() { + if (isNullOrEmpty(remoteHost)) { + throw new ReplicationException("Remote host is a mandatory field"); + } + + if (remotePort == null) { + throw new ReplicationException("Remote port is a mandatory field"); + } + + if (isNullOrEmpty(tenant)) { + throw new ReplicationException("Tenant id is a mandatory field"); + } + + if (db == null) { + throw new ReplicationException("Database is a mandatory field"); + } + + if (collection == null || isNullOrEmpty(collection.getName())) { + throw new ReplicationException("Collection or repository is a mandatory field"); + } + + if (isNullOrEmpty(userName)) { + throw new ReplicationException("Username is a mandatory field"); + } + } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicaChangeListener.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicaChangeListener.java deleted file mode 100644 index f511cb32d..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicaChangeListener.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync; - -import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.collection.events.CollectionEventInfo; -import org.dizitart.no2.collection.events.CollectionEventListener; -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.dizitart.no2.sync.event.ReplicationEvent; -import org.dizitart.no2.sync.event.ReplicationEventType; -import org.dizitart.no2.sync.message.DataGateFeed; - -import java.util.Collections; - -import static org.dizitart.no2.common.Constants.REPLICATOR; - -/** - * @author Anindya Chatterjee - */ -@Slf4j -class ReplicaChangeListener implements CollectionEventListener { - private final ReplicationTemplate replicationTemplate; - private final MessageTemplate messageTemplate; - - public ReplicaChangeListener(ReplicationTemplate replicationTemplate, MessageTemplate messageTemplate) { - this.replicationTemplate = replicationTemplate; - this.messageTemplate = messageTemplate; - } - - @Override - public void onEvent(CollectionEventInfo eventInfo) { - try { - if (eventInfo != null) { - if (!REPLICATOR.equals(eventInfo.getOriginator())) { - switch (eventInfo.getEventType()) { - case Insert: - case Update: - Document document = (Document) eventInfo.getItem(); - handleModifyEvent(document); - break; - case Remove: - document = (Document) eventInfo.getItem(); - handleRemoveEvent(document); - break; - case IndexStart: - case IndexEnd: - break; - } - } - } - } catch (Exception e) { - log.error("Error while processing collection event", e); - replicationTemplate.postEvent(new ReplicationEvent(ReplicationEventType.Error, e)); - } - } - - private void handleRemoveEvent(Document document) { - LastWriteWinState state = new LastWriteWinState(); - NitriteId nitriteId = document.getId(); - Long deleteTime = document.getLastModifiedSinceEpoch(); - - if (replicationTemplate.getCrdt() != null) { - replicationTemplate.getCrdt().getTombstones().put(nitriteId, deleteTime); - state.setTombstones(Collections.singletonMap(nitriteId.getIdValue(), deleteTime)); - sendFeed(state); - } - } - - private void handleModifyEvent(Document document) { - LastWriteWinState state = new LastWriteWinState(); - state.setChanges(Collections.singleton(document)); - sendFeed(state); - } - - private void sendFeed(LastWriteWinState state) { - if (replicationTemplate.shouldExchangeFeed() && messageTemplate != null) { - MessageFactory factory = replicationTemplate.getMessageFactory(); - DataGateFeed feedMessage = factory.createFeedMessage(replicationTemplate.getConfig(), - replicationTemplate.getReplicaId(), state); - - FeedJournal journal = replicationTemplate.getFeedJournal(); - messageTemplate.sendMessage(feedMessage); - journal.write(state); - } - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicatedCollection.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicatedCollection.java new file mode 100644 index 000000000..65613c9ee --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicatedCollection.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.sync; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import okhttp3.WebSocket; +import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.common.meta.Attributes; +import org.dizitart.no2.common.util.StringUtils; +import org.dizitart.no2.sync.crdt.ConflictFreeReplicatedDataType; +import org.dizitart.no2.sync.crdt.LastWriteWinMap; +import org.dizitart.no2.sync.crdt.Markers; +import org.dizitart.no2.sync.event.CollectionChangeListener; +import org.dizitart.no2.sync.handlers.ReceiptLedgerAware; +import org.dizitart.no2.sync.message.BatchMessage; +import org.dizitart.no2.sync.message.Receipt; +import org.dizitart.no2.sync.net.CloseReason; +import org.dizitart.no2.sync.net.DataGateClient; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.dizitart.no2.common.meta.Attributes.REPLICA; + +/** + * @author Anindya Chatterjee + */ +@Slf4j +public class ReplicatedCollection implements ReceiptLedgerAware { + private String replicaId; + private AtomicBoolean stopped; + + @Getter private final Config config; + @Getter private final NitriteCollection collection; + @Getter private FeedLedger feedLedger; + @Getter private DataGateSocketListener dataGateSocketListener; + @Getter private ConflictFreeReplicatedDataType replicatedDataType; + @Getter private BatchChangeSender batchChangeSender; + + public ReplicatedCollection(Config config) { + this.config = config; + this.collection = config.getCollection(); + initialize(); + } + + public void startReplication() { + log.debug("Starting replication for {}", getReplicaId()); + DataGateClient dataGateClient = new DataGateClient(config); + dataGateSocketListener = new DataGateSocketListener(config, this); + batchChangeSender = new BatchChangeSender(config, this, dataGateSocketListener); + replicatedDataType.resetCounter(); + dataGateClient.setListener(dataGateSocketListener); + } + + public void stopReplication(WebSocket webSocket, CloseReason reason) { + dataGateSocketListener.closeConnection(webSocket, reason); + setStopped(true); + } + + public void sendAndReceive(WebSocket webSocket, BatchMessage batchMessage) { + batchChangeSender.sendAndReceive(webSocket, batchMessage); + } + + public void collectGarbage(Long dtl) { + if (dtl != null && dtl > 0) { + long collectTime = System.currentTimeMillis() - dtl * 24 * 60 * 60 * 1000; + + Receipt garbage = replicatedDataType.collectGarbage(collectTime); + feedLedger.writeOff(garbage); + } + } + + public void setStopped(boolean stopped) { + if (this.stopped == null) { + this.stopped = new AtomicBoolean(); + } + this.stopped.set(stopped); + } + + public boolean isStopped() { + return stopped != null && stopped.get(); + } + + public String getReplicaId() { + if (StringUtils.isNullOrEmpty(replicaId)) { + Attributes attributes = collection.getAttributes(); + + if (attributes == null) { + attributes = new Attributes(); + collection.setAttributes(attributes); + } + + if (!attributes.hasKey(Attributes.REPLICA)) { + String name = StringUtils.isNullOrEmpty(config.getReplicaName()) + ? UUID.randomUUID().toString() + : config.getReplicaName() + "[" + UUID.randomUUID() + "]"; + attributes.set(REPLICA, name); + } + replicaId = attributes.get(Attributes.REPLICA); + } + return replicaId; + } + + public void setLocalNextMarkers(Markers markers) { + replicatedDataType.setLocalNextMarkers(markers); + } + + public void setRemoteNextMarkers(Markers markers) { + replicatedDataType.setRemoteNextMarkers(markers); + } + + private void initialize() { + stopped = new AtomicBoolean(true); + replicatedDataType = new LastWriteWinMap(config); + feedLedger = new FeedLedger(config); + CollectionChangeListener changeListener = new CollectionChangeListener(replicatedDataType); + getCollection().subscribe(changeListener); + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationException.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationException.java index 788b84751..e0d541675 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationException.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationException.java @@ -24,6 +24,11 @@ public class ReplicationException extends NitriteException { private final boolean fatal; + public ReplicationException(String errorMessage) { + super(errorMessage); + this.fatal = false; + } + public ReplicationException(String errorMessage, boolean fatal) { super(errorMessage); this.fatal = fatal; diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationOperation.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationOperation.java deleted file mode 100644 index 1fb887063..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationOperation.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync; - -import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.collection.meta.Attributes; -import org.dizitart.no2.common.util.StringUtils; -import org.dizitart.no2.store.NitriteMap; -import org.dizitart.no2.store.NitriteStore; -import org.dizitart.no2.sync.crdt.LastWriteWinMap; - -import java.util.UUID; - -import static org.dizitart.no2.collection.meta.Attributes.LAST_SYNCED; -import static org.dizitart.no2.collection.meta.Attributes.TOMBSTONE; -import static org.dizitart.no2.common.Constants.INTERNAL_NAME_SEPARATOR; - -/** - * @author Anindya Chatterjee - */ -interface ReplicationOperation { - NitriteCollection getCollection(); - - default Attributes getAttributes() { - Attributes attributes = getCollection().getAttributes(); - if (attributes == null) { - attributes = new Attributes(); - saveAttributes(attributes); - } - return attributes; - } - - default void saveAttributes(Attributes attributes) { - getCollection().setAttributes(attributes); - } - - default Long getLastSyncTime() { - Attributes attributes = getAttributes(); - String syncTimeStr = attributes.get(LAST_SYNCED); - if (StringUtils.isNullOrEmpty(syncTimeStr)) { - return Long.MIN_VALUE; - } else { - return Long.parseLong(syncTimeStr); - } - } - - default LastWriteWinMap createReplicatedDataType() { - Attributes attributes = getAttributes(); - String tombstoneName = getTombstoneName(attributes); - saveAttributes(attributes); - - NitriteStore store = getCollection().getStore(); - NitriteMap tombstone = store.openMap(tombstoneName, NitriteId.class, Long.class); - return new LastWriteWinMap(getCollection(), tombstone); - } - - default String getTombstoneName(Attributes attributes) { - String tombstoneName = attributes.get(TOMBSTONE); - if (StringUtils.isNullOrEmpty(tombstoneName)) { - tombstoneName = getCollection().getName() - + INTERNAL_NAME_SEPARATOR + TOMBSTONE - + INTERNAL_NAME_SEPARATOR + UUID.randomUUID(); - attributes.set(TOMBSTONE, tombstoneName); - } - return tombstoneName; - } - - default void saveLastSyncTime(Long lastSyncTime) { - Attributes attributes = getAttributes(); - attributes.set(LAST_SYNCED, Long.toString(lastSyncTime)); - saveAttributes(attributes); - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationTemplate.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationTemplate.java deleted file mode 100644 index ac7feea96..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/ReplicationTemplate.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync; - -import lombok.AccessLevel; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.collection.meta.Attributes; -import org.dizitart.no2.common.tuples.Pair; -import org.dizitart.no2.common.util.StringUtils; -import org.dizitart.no2.sync.crdt.LastWriteWinMap; -import org.dizitart.no2.sync.event.ReplicationEvent; -import org.dizitart.no2.sync.event.ReplicationEventBus; -import org.dizitart.no2.sync.event.ReplicationEventListener; -import org.dizitart.no2.sync.message.Connect; -import org.dizitart.no2.sync.message.Disconnect; -import org.dizitart.no2.sync.message.Receipt; - -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.dizitart.no2.collection.meta.Attributes.REPLICA; -import static org.dizitart.no2.sync.event.ReplicationEventType.Started; -import static org.dizitart.no2.sync.event.ReplicationEventType.Stopped; - -/** - * @author Anindya Chatterjee - */ -@Slf4j -@Getter -public class ReplicationTemplate implements ReplicationOperation { - private final Config config; - private MessageFactory messageFactory; - private MessageTemplate messageTemplate; - private LastWriteWinMap crdt; - private FeedJournal feedJournal; - - @Getter(AccessLevel.NONE) - private BatchChangeScheduler batchChangeScheduler; - - @Getter(AccessLevel.NONE) - private String replicaId; - - @Getter(AccessLevel.NONE) - private ReplicaChangeListener replicaChangeListener; - - @Getter(AccessLevel.NONE) - private AtomicBoolean connected; - - @Getter(AccessLevel.NONE) - private AtomicBoolean exchangeFlag; - - @Getter(AccessLevel.NONE) - private AtomicBoolean acceptCheckpoint; - - @Getter(AccessLevel.NONE) - private ReplicationEventBus eventBus; - - public ReplicationTemplate(Config config) { - this.config = config; - initialize(); - } - - public void connect() { - messageTemplate.openConnection(); - Connect message = messageFactory.createConnect(config, getReplicaId()); - messageTemplate.sendMessage(message); - eventBus.post(new ReplicationEvent(Started)); - } - - public void setConnected() { - connected.compareAndSet(false, true); - } - - public boolean isConnected() { - return connected.get(); - } - - public void disconnect() { - Disconnect message = messageFactory.createDisconnect(config, getReplicaId()); - messageTemplate.sendMessage(message); - stopReplication("User disconnect"); - } - - public void stopReplication(String reason) { - batchChangeScheduler.stop(); - eventBus.post(new ReplicationEvent(Stopped)); - connected.set(false); - exchangeFlag.set(false); - acceptCheckpoint.set(false); - messageTemplate.closeConnection(reason); - } - - public void sendChanges() { - batchChangeScheduler.schedule(); - } - - public void startFeedExchange() { - this.exchangeFlag.compareAndSet(false, true); - } - - public boolean shouldExchangeFeed() { - return exchangeFlag.get(); - } - - public String getReplicaId() { - if (StringUtils.isNullOrEmpty(replicaId)) { - Attributes attributes = getAttributes(); - if (!attributes.hasKey(Attributes.REPLICA)) { - attributes.set(REPLICA, UUID.randomUUID().toString()); - } - replicaId = attributes.get(Attributes.REPLICA); - } - return replicaId; - } - - public void setAcceptCheckpoint() { - acceptCheckpoint.compareAndSet(false, true); - } - - public boolean shouldAcceptCheckpoint() { - return acceptCheckpoint.get(); - } - - @Override - public NitriteCollection getCollection() { - return config.getCollection(); - } - - public void subscribe(ReplicationEventListener listener) { - eventBus.register(listener); - } - - public void unsubscribe(ReplicationEventListener listener) { - eventBus.deregister(listener); - } - - public void postEvent(ReplicationEvent event) { - eventBus.post(event); - } - - public void close() { - eventBus.close(); - messageTemplate.close(); - batchChangeScheduler.stop(); - this.getCollection().unsubscribe(replicaChangeListener); - } - - public void collectGarbage(Long ttl) { - if (ttl != null && ttl > 0) { - long collectTime = System.currentTimeMillis() - ttl; - if (crdt != null && crdt.getTombstones() != null) { - Set removeSet = new HashSet<>(); - for (Pair entry : crdt.getTombstones().entries()) { - if (entry.getSecond() < collectTime) { - removeSet.add(entry.getFirst()); - } - } - - Receipt garbage = new Receipt(); - for (NitriteId nitriteId : removeSet) { - crdt.getTombstones().remove(nitriteId); - garbage.getRemoved().add(nitriteId.getIdValue()); - } - - feedJournal.accumulate(garbage); - } - } - } - - private void initialize() { - this.messageFactory = new MessageFactory(); - this.connected = new AtomicBoolean(false); - this.exchangeFlag = new AtomicBoolean(false); - this.acceptCheckpoint = new AtomicBoolean(false); - this.eventBus = new ReplicationEventBus(); - this.messageTemplate = new MessageTemplate(config, this); - this.crdt = createReplicatedDataType(); - this.feedJournal = new FeedJournal(this); - this.batchChangeScheduler = new BatchChangeScheduler(this); - this.replicaChangeListener = new ReplicaChangeListener(this, messageTemplate); - this.getCollection().subscribe(replicaChangeListener); - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/ConflictFreeReplicatedDataType.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/ConflictFreeReplicatedDataType.java new file mode 100644 index 000000000..2773e31e2 --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/ConflictFreeReplicatedDataType.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.sync.crdt; + +import org.dizitart.no2.Nitrite; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.collection.NitriteId; +import org.dizitart.no2.common.meta.Attributes; +import org.dizitart.no2.common.SortOrder; +import org.dizitart.no2.common.util.StringUtils; +import org.dizitart.no2.filters.Filter; +import org.dizitart.no2.index.IndexType; +import org.dizitart.no2.sync.Config; +import org.dizitart.no2.sync.message.Receipt; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.dizitart.no2.collection.FindOptions.orderBy; +import static org.dizitart.no2.common.meta.Attributes.*; +import static org.dizitart.no2.common.Constants.DOC_MODIFIED; +import static org.dizitart.no2.filters.FluentFilter.where; +import static org.dizitart.no2.index.IndexOptions.indexOptions; + +/** + * @author Anindya Chatterjee + */ +public abstract class ConflictFreeReplicatedDataType implements AutoCloseable { + protected static final String TOMBSTONE_COUNTER = "tombstone_counter"; + protected static final String DELETED_TIME = "deleted_time"; + protected static final String TOMBSTONE_SOURCE = "tombstone_source"; + + protected final Config config; + protected NitriteCollection collection; + protected NitriteCollection tombstones; + protected AtomicInteger counter; + + public abstract void createTombstone(NitriteId nitriteId, Long deleteTime); + public abstract void merge(DeltaStates deltaStates); + public abstract DeltaStates delta(Markers startMarker, Markers endMarker, + int offset, int size); + + protected ConflictFreeReplicatedDataType(Config config) { + this.config = config; + initializeDataType(); + } + + public void resetCounter() { + this.counter = new AtomicInteger(0); + } + + public Long getTombstoneTime(NitriteId id) { + Document tombstone = tombstones.getById(id); + if (tombstone == null) return 0L; + return tombstone.get(DELETED_TIME, Long.class); + } + + public Document getDocument(NitriteId id) { + return collection.getById(id); + } + + public Markers getLocalEndMarkers() { + Markers markers = new Markers(); + + // get last updated document from DOC_MODIFIED index and get its modified time + Document latest = collection.find(orderBy(DOC_MODIFIED, SortOrder.Descending)).firstOrNull(); + markers.setCollectionMarker(latest == null ? 0 : latest.getLastModifiedSinceEpoch()); + + // get the highest tombstone counter for the current collection + Document latestTombstone = tombstones.find( + where(TOMBSTONE_SOURCE).eq(collection.getName()), + orderBy(TOMBSTONE_COUNTER, SortOrder.Descending)).firstOrNull(); + + markers.setTombstoneMarker(latestTombstone == null ? 0L : latestTombstone.get(TOMBSTONE_COUNTER, Long.class)); + + return markers; + } + + public Receipt collectGarbage(long collectTime) { + Set removeSet = new HashSet<>(); + for (Document entry : tombstones.find()) { + if (entry.get(TOMBSTONE_COUNTER, Long.class) < collectTime) { + removeSet.add(entry.getId()); + } + } + + Receipt garbage = new Receipt(); + for (NitriteId nitriteId : removeSet) { + tombstones.remove(Filter.byId(nitriteId)); + garbage.getRemoved().add(nitriteId.getIdValue()); + } + + return garbage; + } + + @Override + public void close() { + // collection should not be closed as it may be used outside of replication + // but as tombstone is only used in replication, so close tombstone. + if (tombstones != null) { + tombstones.close(); + } + } + + public Markers getLocalStartMarkers() { + return getMarkers(LOCAL_COLLECTION_MARKER, LOCAL_TOMBSTONE_MARKER); + } + + public void setLocalNextMarkers(Markers markers) { + setMarkers(LOCAL_COLLECTION_MARKER, LOCAL_TOMBSTONE_MARKER, markers); + } + + public Markers getRemoteStartMarkers() { + return getMarkers(REMOTE_COLLECTION_MARKER, REMOTE_TOMBSTONE_MARKER); + } + + public void setRemoteNextMarkers(Markers markers) { + setMarkers(REMOTE_COLLECTION_MARKER, REMOTE_TOMBSTONE_MARKER, markers); + } + + public Attributes getCollectionAttributes() { + Attributes attributes = collection.getAttributes(); + + if (attributes == null) { + attributes = new Attributes(); + collection.setAttributes(attributes); + } + return attributes; + } + + public Attributes getTombstoneAttributes() { + Attributes attributes = tombstones.getAttributes(); + + if (attributes == null) { + attributes = new Attributes(); + tombstones.setAttributes(attributes); + } + return attributes; + } + + private void initializeDataType() { + this.collection = config.getCollection(); + this.counter = new AtomicInteger(0); + + Nitrite db = config.getDb(); + Attributes collectionAttributes = getCollectionAttributes(); + String tombstoneName = getTombstoneName(collectionAttributes); + + this.tombstones = db.getCollection(tombstoneName); + ensureIndices(); + } + + private String getTombstoneName(Attributes attributes) { + String tombstoneName = attributes.get(TOMBSTONE); + if (StringUtils.isNullOrEmpty(tombstoneName)) { + tombstoneName = collection.getName() + "_" + TOMBSTONE; + attributes.set(TOMBSTONE, tombstoneName); + } + return tombstoneName; + } + + private Markers getMarkers(String collectionKey, String tombstoneKey) { + Markers markers = new Markers(); + + String collectionMarker = getCollectionAttributes().get(collectionKey); + if (StringUtils.isNullOrEmpty(collectionMarker)) { + markers.setCollectionMarker(0L); + } else { + markers.setCollectionMarker(Long.parseLong(collectionMarker)); + } + + String tombstoneMarker = getTombstoneAttributes().get(tombstoneKey); + if (StringUtils.isNullOrEmpty(tombstoneMarker)) { + markers.setTombstoneMarker(0L); + } else { + markers.setTombstoneMarker(Long.parseLong(tombstoneMarker)); + } + + return markers; + } + + private void setMarkers(String collectionKey, String tombstoneKey, Markers markers) { + Attributes collectionAttributes = getCollectionAttributes(); + collectionAttributes.set(collectionKey, Objects.toString(markers.getCollectionMarker())); + collection.setAttributes(collectionAttributes); + + Attributes tombstoneAttributes = getTombstoneAttributes(); + tombstoneAttributes.set(tombstoneKey, Objects.toString(markers.getTombstoneMarker())); + tombstones.setAttributes(tombstoneAttributes); + } + + private void ensureIndices() { + if (!collection.hasIndex(DOC_MODIFIED)) { + collection.createIndex(indexOptions(IndexType.NON_UNIQUE), DOC_MODIFIED); + } + + if (!tombstones.hasIndex(TOMBSTONE_COUNTER)) { + tombstones.createIndex(indexOptions(IndexType.NON_UNIQUE), TOMBSTONE_COUNTER); + } + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/LastWriteWinState.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/DeltaStates.java similarity index 82% rename from nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/LastWriteWinState.java rename to nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/DeltaStates.java index 282350085..084dd1de9 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/LastWriteWinState.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/DeltaStates.java @@ -30,13 +30,13 @@ * @author Anindya Chatterjee */ @Data -public class LastWriteWinState { +public class DeltaStates { @JsonDeserialize(contentUsing = DocumentDeserializer.class) - private Set changes; - private Map tombstones; + private Set changeSet; + private Map tombstoneMap; - public LastWriteWinState() { - changes = new LinkedHashSet<>(); - tombstones = new LinkedHashMap<>(); + public DeltaStates() { + changeSet = new LinkedHashSet<>(); + tombstoneMap = new LinkedHashMap<>(); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/LastWriteWinMap.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/LastWriteWinMap.java index 4e011ebec..3dd3fa261 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/LastWriteWinMap.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/LastWriteWinMap.java @@ -16,61 +16,90 @@ package org.dizitart.no2.sync.crdt; -import lombok.Data; -import org.dizitart.no2.collection.*; -import org.dizitart.no2.common.tuples.Pair; -import org.dizitart.no2.store.NitriteMap; - +import lombok.extern.slf4j.Slf4j; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.collection.DocumentCursor; +import org.dizitart.no2.collection.NitriteId; +import org.dizitart.no2.filters.Filter; +import org.dizitart.no2.sync.Config; + +import java.util.HashMap; import java.util.Map; +import java.util.Set; import static org.dizitart.no2.collection.FindOptions.skipBy; import static org.dizitart.no2.common.Constants.*; +import static org.dizitart.no2.filters.Filter.and; import static org.dizitart.no2.filters.FluentFilter.where; /** * @author Anindya Chatterjee. */ -@Data -public class LastWriteWinMap { - private NitriteCollection collection; - private NitriteMap tombstones; - - public LastWriteWinMap(NitriteCollection collection, NitriteMap tombstones) { - this.collection = collection; - this.tombstones = tombstones; +@Slf4j +public class LastWriteWinMap extends ConflictFreeReplicatedDataType { + public LastWriteWinMap(Config config) { + super(config); + } + + @Override + public void createTombstone(NitriteId nitriteId, Long deleteTime) { + if (tombstones != null) { + writeTombstoneEntry(nitriteId, deleteTime, collection.getName()); + } } - public void merge(LastWriteWinState snapshot) { - if (snapshot.getChanges() != null) { - for (Document entry : snapshot.getChanges()) { + @Override + public void merge(DeltaStates deltaStates) { + if (deltaStates.getChangeSet() != null) { + for (Document entry : deltaStates.getChangeSet()) { put(entry); } } - if (snapshot.getTombstones() != null) { - for (Map.Entry entry : snapshot.getTombstones().entrySet()) { + if (deltaStates.getTombstoneMap() != null) { + for (Map.Entry entry : deltaStates.getTombstoneMap().entrySet()) { remove(NitriteId.createId(entry.getKey()), entry.getValue()); } } } - public LastWriteWinState getChangesSince(Long since, int offset, int size) { - LastWriteWinState state = new LastWriteWinState(); + @Override + public DeltaStates delta(Markers startMarker, Markers endMarker, int offset, int size) { + DeltaStates deltaStates = new DeltaStates(); + deltaStates.setChangeSet(getDocumentChanges(startMarker.getCollectionMarker(), endMarker.getCollectionMarker(), + offset, size)); + deltaStates.setTombstoneMap(getTombstoneChanges(startMarker.getTombstoneMarker(), endMarker.getTombstoneMarker(), + offset, size)); - DocumentCursor cursor = collection.find(where(DOC_MODIFIED).gte(since), skipBy(offset).limit(size)); - state.getChanges().addAll(cursor.toSet()); + return deltaStates; + } - if (offset == 0) { - // don't repeat for other offsets - for (Pair entry : tombstones.entries()) { - Long timestamp = entry.getSecond(); - if (timestamp >= since) { - state.getTombstones().put(entry.getFirst().getIdValue(), entry.getSecond()); - } - } - } + private Set getDocumentChanges(Long startTime, Long endTime, + int offset, int size) { + DocumentCursor cursor = collection.find( + and( + where(DOC_MODIFIED).gt(startTime), + where(DOC_MODIFIED).lte(endTime) + ), skipBy(offset).limit(size)); - return state; + return cursor.toSet(); + } + + private Map getTombstoneChanges(Long startTime, Long endTime, + int offset, int size) { + DocumentCursor cursor = tombstones.find( + and( + where(TOMBSTONE_COUNTER).gt(startTime), + where(TOMBSTONE_COUNTER).lte(endTime), + where(TOMBSTONE_SOURCE).eq(collection.getName()) + ), skipBy(offset).limit(size)); + + Map tombstoneMap = new HashMap<>(); + for (Document entry : cursor) { + Long deletedTime = entry.get(DELETED_TIME, Long.class); + tombstoneMap.put(entry.getId().getIdValue(), deletedTime); + } + return tombstoneMap; } private void put(Document value) { @@ -79,29 +108,45 @@ private void put(Document value) { Document entry = collection.getById(key); if (entry == null) { - if (tombstones.containsKey(key)) { - Long tombstoneTime = tombstones.get(key); + Document tombstone = tombstones.getById(key); + if (tombstone != null) { + Long tombstoneTime = tombstone.get(TOMBSTONE_COUNTER, Long.class); Long docModifiedTime = value.getLastModifiedSinceEpoch(); if (docModifiedTime >= tombstoneTime) { value.put(DOC_SOURCE, REPLICATOR); collection.insert(value); - tombstones.remove(key); + + destroyTombstone(key); } } else { value.put(DOC_SOURCE, REPLICATOR); collection.insert(value); } } else { - Long oldTime = entry.getLastModifiedSinceEpoch(); - Long newTime = value.getLastModifiedSinceEpoch(); + Integer entryRevision = entry.getRevision(); + Integer valueRevision = value.getRevision(); - if (newTime > oldTime) { + if (valueRevision > entryRevision) { + // if the document revision is higher update it entry.put(DOC_SOURCE, REPLICATOR); collection.remove(entry); value.put(DOC_SOURCE, REPLICATOR); collection.insert(value); + } else if (valueRevision.equals(entryRevision)) { + // in case for same revision number, check the last modified time + // if the new document is latest, update it + Long oldTime = entry.getLastModifiedSinceEpoch(); + Long newTime = value.getLastModifiedSinceEpoch(); + + if (newTime > oldTime) { + entry.put(DOC_SOURCE, REPLICATOR); + collection.remove(entry); + + value.put(DOC_SOURCE, REPLICATOR); + collection.insert(value); + } } } } @@ -112,7 +157,25 @@ private void remove(NitriteId key, long timestamp) { if (entry != null) { entry.put(DOC_SOURCE, REPLICATOR); collection.remove(entry); - tombstones.put(key, timestamp); + + writeTombstoneEntry(key, timestamp, REPLICATOR); } } + + private void writeTombstoneEntry(NitriteId nitriteId, Long deleteTime, String source) { + // to make the counter unique for each item for bulk delete + long tombstoneCounter = REPLICATOR.equalsIgnoreCase(source) + ? 0 : System.currentTimeMillis() + counter.incrementAndGet(); + Document tombstone = Document + .createDocument(DOC_ID, nitriteId.getIdValue()) + .put(DELETED_TIME, deleteTime) + .put(TOMBSTONE_COUNTER, tombstoneCounter) + .put(TOMBSTONE_SOURCE, source); + + tombstones.insert(tombstone); + } + + private void destroyTombstone(NitriteId nitriteId) { + tombstones.remove(Filter.byId(nitriteId)); + } } diff --git a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdExtensionTest.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/Markers.java similarity index 68% rename from nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdExtensionTest.java rename to nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/Markers.java index 2a02ad524..dbf3a326b 100644 --- a/nitrite-jackson-mapper/src/test/java/org/dizitart/no2/common/mapper/extensions/NitriteIdExtensionTest.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/Markers.java @@ -15,15 +15,15 @@ * */ -package org.dizitart.no2.common.mapper.extensions; +package org.dizitart.no2.sync.crdt; -import org.junit.Assert; -import org.junit.Test; +import lombok.Data; -public class NitriteIdExtensionTest { - @Test - public void testGetSupportedTypes() { - Assert.assertEquals(1, (new NitriteIdExtension()).getSupportedTypes().size()); - } +/** + * @author Anindya Chatterjee + */ +@Data +public class Markers { + private Long collectionMarker; + private Long tombstoneMarker; } - diff --git a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/compat/v3/MVMapBuilderTest.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/Tombstone.java similarity index 58% rename from nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/compat/v3/MVMapBuilderTest.java rename to nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/Tombstone.java index 9488930dc..15856d5b4 100644 --- a/nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/compat/v3/MVMapBuilderTest.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/crdt/Tombstone.java @@ -15,18 +15,17 @@ * */ -package org.dizitart.no2.mvstore.compat.v3; +package org.dizitart.no2.sync.crdt; -import org.junit.Test; +import lombok.Data; +import org.dizitart.no2.collection.NitriteId; -import static org.junit.Assert.assertTrue; - -public class MVMapBuilderTest { - @Test - public void testConstructor() { - MVMapBuilder actualMvMapBuilder = new MVMapBuilder<>(); - assertTrue(actualMvMapBuilder.getKeyType() instanceof NitriteDataType); - assertTrue(actualMvMapBuilder.getValueType() instanceof NitriteDataType); - } +/** + * @author Anindya Chatterjee + */ +@Data +public class Tombstone { + private NitriteId nitriteId; + private Long deleteTimestamp; + private Long syncTimestamp; } - diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/event/CollectionChangeListener.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/event/CollectionChangeListener.java new file mode 100644 index 000000000..2cb8bfed3 --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/event/CollectionChangeListener.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.sync.event; + +import lombok.extern.slf4j.Slf4j; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.collection.NitriteId; +import org.dizitart.no2.collection.events.CollectionEventInfo; +import org.dizitart.no2.collection.events.CollectionEventListener; +import org.dizitart.no2.sync.crdt.ConflictFreeReplicatedDataType; + +import static org.dizitart.no2.common.Constants.REPLICATOR; + +/** + * @author Anindya Chatterjee + */ +@Slf4j +public class CollectionChangeListener implements CollectionEventListener { + private final ConflictFreeReplicatedDataType replicatedDataType; + + public CollectionChangeListener(ConflictFreeReplicatedDataType replicatedDataType) { + this.replicatedDataType = replicatedDataType; + } + + @Override + public void onEvent(CollectionEventInfo eventInfo) { + if (eventInfo != null) { + if (!REPLICATOR.equals(eventInfo.getOriginator())) { + // discard the removes coming from replicator crdt + switch (eventInfo.getEventType()) { + case Remove: + Document document = (Document) eventInfo.getItem(); + handleRemoveEvent(document); + break; + case Insert: + case Update: + case IndexStart: + case IndexEnd: + break; + } + } + } + } + + private void handleRemoveEvent(Document document) { + NitriteId nitriteId = document.getId(); + Long deleteTime = document.getLastModifiedSinceEpoch(); + + if (replicatedDataType != null) { + replicatedDataType.createTombstone(nitriteId, deleteTime); + } + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchAckHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchAckHandler.java index c6a808d8d..302f5c039 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchAckHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchAckHandler.java @@ -16,8 +16,9 @@ package org.dizitart.no2.sync.handlers; -import org.dizitart.no2.sync.FeedJournal; -import org.dizitart.no2.sync.ReplicationTemplate; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.FeedLedger; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.BatchAck; import org.dizitart.no2.sync.message.Receipt; @@ -25,16 +26,17 @@ * @author Anindya Chatterjee */ public class BatchAckHandler implements MessageHandler { - private final ReplicationTemplate replicationTemplate; + private final ReplicatedCollection replicatedCollection; - public BatchAckHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public BatchAckHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(BatchAck message) { + public void handleMessage(WebSocket webSocket, BatchAck message) { Receipt receipt = message.getReceipt(); - FeedJournal journal = replicationTemplate.getFeedJournal(); - journal.accumulate(receipt); + FeedLedger feedLedger = replicatedCollection.getFeedLedger(); + feedLedger.writeOff(receipt); + replicatedCollection.sendAndReceive(webSocket, message); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeContinueHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeContinueHandler.java index 3a21faede..2825bc829 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeContinueHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeContinueHandler.java @@ -16,9 +16,10 @@ package org.dizitart.no2.sync.handlers; -import lombok.Data; +import lombok.Getter; +import okhttp3.WebSocket; import org.dizitart.no2.sync.MessageFactory; -import org.dizitart.no2.sync.ReplicationTemplate; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.BatchAck; import org.dizitart.no2.sync.message.BatchChangeContinue; import org.dizitart.no2.sync.message.Receipt; @@ -26,23 +27,22 @@ /** * @author Anindya Chatterjee */ -@Data public class BatchChangeContinueHandler implements MessageHandler, ReceiptAckSender { - private ReplicationTemplate replicationTemplate; + @Getter private final ReplicatedCollection replicatedCollection; - public BatchChangeContinueHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public BatchChangeContinueHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(BatchChangeContinue message) { - sendAck(message); + public void handleMessage(WebSocket webSocket, BatchChangeContinue message) { + sendAck(webSocket, message); } @Override - public BatchAck createAck(String correlationId, Receipt receipt) { - MessageFactory factory = replicationTemplate.getMessageFactory(); - return factory.createBatchAck(replicationTemplate.getConfig(), - correlationId, replicationTemplate.getReplicaId(), receipt); + public BatchAck createAck(String transactionId, Receipt receipt) { + MessageFactory factory = new MessageFactory(); + return factory.createBatchAck(replicatedCollection.getConfig(), + replicatedCollection.getReplicaId(), transactionId, receipt); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeEndHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeEndHandler.java index 41cdd9d12..da7271801 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeEndHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeEndHandler.java @@ -16,32 +16,35 @@ package org.dizitart.no2.sync.handlers; +import lombok.extern.slf4j.Slf4j; +import okhttp3.WebSocket; import org.dizitart.no2.sync.MessageFactory; -import org.dizitart.no2.sync.MessageTemplate; -import org.dizitart.no2.sync.ReplicationTemplate; +import org.dizitart.no2.sync.DataGateSocketListener; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.BatchChangeEnd; import org.dizitart.no2.sync.message.BatchEndAck; /** * @author Anindya Chatterjee */ +@Slf4j public class BatchChangeEndHandler implements MessageHandler { - private final ReplicationTemplate replicationTemplate; + private final ReplicatedCollection replicatedCollection; - public BatchChangeEndHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public BatchChangeEndHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(BatchChangeEnd message) { - MessageFactory factory = replicationTemplate.getMessageFactory(); - BatchEndAck batchEndAck = factory.createBatchEndAck(replicationTemplate.getConfig(), - replicationTemplate.getReplicaId(), message.getHeader().getId()); + public void handleMessage(WebSocket webSocket, BatchChangeEnd message) { + MessageFactory factory = new MessageFactory(); + BatchEndAck batchEndAck = factory.createBatchEndAck(replicatedCollection.getConfig(), + replicatedCollection.getReplicaId(), message.getHeader().getTransactionId()); + batchEndAck.getHeader().setCorrelationId(message.getHeader().getId()); - MessageTemplate messageTemplate = replicationTemplate.getMessageTemplate(); - messageTemplate.sendMessage(batchEndAck); - Long time = message.getHeader().getTimestamp(); - replicationTemplate.saveLastSyncTime(time); - replicationTemplate.setAcceptCheckpoint(); + DataGateSocketListener dataGateSocketListener = replicatedCollection.getDataGateSocketListener(); + dataGateSocketListener.sendMessage(webSocket, batchEndAck); + + replicatedCollection.setRemoteNextMarkers(message.getEndMarkers()); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeStartHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeStartHandler.java index 7575ab818..ceb3bf316 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeStartHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchChangeStartHandler.java @@ -16,9 +16,10 @@ package org.dizitart.no2.sync.handlers; -import lombok.Data; +import lombok.Getter; +import okhttp3.WebSocket; import org.dizitart.no2.sync.MessageFactory; -import org.dizitart.no2.sync.ReplicationTemplate; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.BatchAck; import org.dizitart.no2.sync.message.BatchChangeStart; import org.dizitart.no2.sync.message.Receipt; @@ -26,23 +27,22 @@ /** * @author Anindya Chatterjee */ -@Data public class BatchChangeStartHandler implements MessageHandler, ReceiptAckSender { - private ReplicationTemplate replicationTemplate; + @Getter private final ReplicatedCollection replicatedCollection; - public BatchChangeStartHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public BatchChangeStartHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(BatchChangeStart message) { - sendAck(message); + public void handleMessage(WebSocket webSocket, BatchChangeStart message) { + sendAck(webSocket, message); } @Override - public BatchAck createAck(String correlationId, Receipt receipt) { - MessageFactory factory = replicationTemplate.getMessageFactory(); - return factory.createBatchAck(replicationTemplate.getConfig(), - replicationTemplate.getReplicaId(), correlationId, receipt); + public BatchAck createAck(String transactionId, Receipt receipt) { + MessageFactory factory = new MessageFactory(); + return factory.createBatchAck(replicatedCollection.getConfig(), + replicatedCollection.getReplicaId(), transactionId, receipt); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchEndAckHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchEndAckHandler.java index 37ee84a08..49be97ffe 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchEndAckHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/BatchEndAckHandler.java @@ -17,24 +17,23 @@ package org.dizitart.no2.sync.handlers; import lombok.Getter; -import org.dizitart.no2.sync.ReplicationTemplate; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.BatchEndAck; -import org.dizitart.no2.sync.message.Receipt; /** * @author Anindya Chatterjee */ @Getter -public class BatchEndAckHandler implements MessageHandler, JournalAware { - private final ReplicationTemplate replicationTemplate; +public class BatchEndAckHandler implements MessageHandler { + private final ReplicatedCollection replicatedCollection; - public BatchEndAckHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public BatchEndAckHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(BatchEndAck message) { - Receipt finalReceipt = getJournal().getFinalReceipt(); - retryFailed(finalReceipt); + public void handleMessage(WebSocket webSocket, BatchEndAck message) { + replicatedCollection.setLocalNextMarkers(message.getEndMarkers()); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ConnectAckHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ConnectAckHandler.java index ca528cd52..57638bd14 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ConnectAckHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ConnectAckHandler.java @@ -17,7 +17,8 @@ package org.dizitart.no2.sync.handlers; import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.sync.ReplicationTemplate; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.ConnectAck; /** @@ -25,17 +26,16 @@ */ @Slf4j public class ConnectAckHandler implements MessageHandler { - private final ReplicationTemplate replica; + private final ReplicatedCollection replicatedCollection; - public ConnectAckHandler(ReplicationTemplate replica) { - this.replica = replica; + public ConnectAckHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(ConnectAck message) { - replica.collectGarbage(message.getTombstoneTtl()); - replica.setConnected(); - replica.startFeedExchange(); - replica.sendChanges(); + public void handleMessage(WebSocket webSocket, ConnectAck message) { + replicatedCollection.collectGarbage(message.getTombstoneTtl()); + replicatedCollection.setStopped(false); + replicatedCollection.sendAndReceive(webSocket, message); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedAckHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedAckHandler.java index c2ab9dec9..e0bee23e8 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedAckHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedAckHandler.java @@ -17,31 +17,24 @@ package org.dizitart.no2.sync.handlers; import lombok.Getter; -import org.dizitart.no2.sync.ReplicationTemplate; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.DataGateFeedAck; import org.dizitart.no2.sync.message.Receipt; /** * @author Anindya Chatterjee */ -@Getter -public class DataGateFeedAckHandler implements MessageHandler, JournalAware { - private final ReplicationTemplate replicationTemplate; +public class DataGateFeedAckHandler implements MessageHandler { + @Getter private final ReplicatedCollection replicatedCollection; - public DataGateFeedAckHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public DataGateFeedAckHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(DataGateFeedAck message) { + public void handleMessage(WebSocket webSocket, DataGateFeedAck message) { Receipt receipt = message.getReceipt(); - - Receipt finalReceipt = getJournal().accumulate(receipt); - retryFailed(finalReceipt); - - if (replicationTemplate.shouldAcceptCheckpoint()) { - Long time = message.getHeader().getTimestamp(); - replicationTemplate.saveLastSyncTime(time); - } + replicatedCollection.getFeedLedger().writeOff(receipt); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedHandler.java index 92b37da52..992cd2f44 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DataGateFeedHandler.java @@ -16,9 +16,10 @@ package org.dizitart.no2.sync.handlers; -import lombok.Data; +import lombok.Getter; +import okhttp3.WebSocket; import org.dizitart.no2.sync.MessageFactory; -import org.dizitart.no2.sync.ReplicationTemplate; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.DataGateFeed; import org.dizitart.no2.sync.message.DataGateFeedAck; import org.dizitart.no2.sync.message.Receipt; @@ -26,27 +27,22 @@ /** * @author Anindya Chatterjee */ -@Data public class DataGateFeedHandler implements MessageHandler, ReceiptAckSender { - private ReplicationTemplate replicationTemplate; + @Getter private final ReplicatedCollection replicatedCollection; - public DataGateFeedHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public DataGateFeedHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(DataGateFeed message) { - sendAck(message); - if (replicationTemplate.shouldAcceptCheckpoint()) { - Long time = message.getHeader().getTimestamp(); - replicationTemplate.saveLastSyncTime(time); - } + public void handleMessage(WebSocket webSocket, DataGateFeed message) { + sendAck(webSocket, message); } @Override - public DataGateFeedAck createAck(String correlationId, Receipt receipt) { - MessageFactory factory = replicationTemplate.getMessageFactory(); - return factory.createFeedAck(replicationTemplate.getConfig(), - replicationTemplate.getReplicaId(), correlationId, receipt); + public DataGateFeedAck createAck(String transactionId, Receipt receipt) { + MessageFactory factory = new MessageFactory(); + return factory.createFeedAck(replicatedCollection.getConfig(), + replicatedCollection.getReplicaId(), transactionId, receipt); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DisconnectHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DisconnectHandler.java index f07d752f1..89399e61b 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DisconnectHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/DisconnectHandler.java @@ -16,21 +16,26 @@ package org.dizitart.no2.sync.handlers; -import org.dizitart.no2.sync.ReplicationTemplate; +import lombok.extern.slf4j.Slf4j; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.ReplicatedCollection; import org.dizitart.no2.sync.message.Disconnect; +import org.dizitart.no2.sync.net.CloseReason; /** * @author Anindya Chatterjee */ +@Slf4j public class DisconnectHandler implements MessageHandler { - private final ReplicationTemplate replicationTemplate; + private final ReplicatedCollection replicatedCollection; - public DisconnectHandler(ReplicationTemplate replicationTemplate) { - this.replicationTemplate = replicationTemplate; + public DisconnectHandler(ReplicatedCollection replicatedCollection) { + this.replicatedCollection = replicatedCollection; } @Override - public void handleMessage(Disconnect message) { - replicationTemplate.stopReplication("Server disconnect"); + public void handleMessage(WebSocket webSocket, Disconnect message) { + log.debug("Disconnecting from server"); + replicatedCollection.stopReplication(webSocket, CloseReason.ServerClose); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ErrorHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ErrorHandler.java index a2f5cfaca..fea9349ca 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ErrorHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ErrorHandler.java @@ -17,7 +17,8 @@ package org.dizitart.no2.sync.handlers; import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.sync.ReplicationTemplate; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.ReplicationException; import org.dizitart.no2.sync.message.ErrorMessage; /** @@ -25,15 +26,10 @@ */ @Slf4j public class ErrorHandler implements MessageHandler { - private final ReplicationTemplate replica; - - public ErrorHandler(ReplicationTemplate replica) { - this.replica = replica; - } @Override - public void handleMessage(ErrorMessage message) { + public void handleMessage(WebSocket webSocket, ErrorMessage message) { log.error("Received error message from server - {}", message.getError()); - replica.stopReplication(message.getError()); + throw new ReplicationException(message.getError(), true); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/JournalAware.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/JournalAware.java deleted file mode 100644 index 6f97dfadb..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/JournalAware.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync.handlers; - -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.sync.FeedJournal; -import org.dizitart.no2.sync.MessageFactory; -import org.dizitart.no2.sync.MessageTemplate; -import org.dizitart.no2.sync.ReplicationTemplate; -import org.dizitart.no2.sync.crdt.LastWriteWinMap; -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.dizitart.no2.sync.message.DataGateFeed; -import org.dizitart.no2.sync.message.Receipt; - -import java.util.HashMap; -import java.util.HashSet; - -/** - * @author Anindya Chatterjee - */ -public interface JournalAware { - ReplicationTemplate getReplicationTemplate(); - - default FeedJournal getJournal() { - return getReplicationTemplate().getFeedJournal(); - } - - default void retryFailed(Receipt receipt) { - if (shouldRetry(receipt)) { - LastWriteWinState state = createState(receipt); - - MessageFactory factory = getReplicationTemplate().getMessageFactory(); - DataGateFeed feedMessage = factory.createFeedMessage(getReplicationTemplate().getConfig(), - getReplicationTemplate().getReplicaId(), state); - - MessageTemplate messageTemplate = getReplicationTemplate().getMessageTemplate(); - messageTemplate.sendMessage(feedMessage); - } - } - - default LastWriteWinState createState(Receipt receipt) { - LastWriteWinState state = new LastWriteWinState(); - state.setTombstones(new HashMap<>()); - state.setChanges(new HashSet<>()); - - NitriteCollection collection = getReplicationTemplate().getCollection(); - LastWriteWinMap crdt = getReplicationTemplate().getCrdt(); - - if (receipt != null) { - if (receipt.getAdded() != null) { - for (String id : receipt.getAdded()) { - Document document = collection.getById(NitriteId.createId(id)); - if (document != null) { - state.getChanges().add(document); - } - } - } - - if (receipt.getRemoved() != null) { - for (String id : receipt.getRemoved()) { - Long timestamp = crdt.getTombstones().get(NitriteId.createId(id)); - if (timestamp != null) { - state.getTombstones().put(id, timestamp); - } - } - } - } - - return state; - } - - default boolean shouldRetry(Receipt receipt) { - if (receipt == null) return false; - if (receipt.getAdded() == null) return false; - if (receipt.getAdded() == null && receipt.getRemoved() == null) return false; - return !receipt.getAdded().isEmpty() || !receipt.getRemoved().isEmpty(); - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/MessageHandler.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/MessageHandler.java index 3c1ba3a5c..73e0bef70 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/MessageHandler.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/MessageHandler.java @@ -16,11 +16,12 @@ package org.dizitart.no2.sync.handlers; +import okhttp3.WebSocket; import org.dizitart.no2.sync.message.DataGateMessage; /** * @author Anindya Chatterjee */ public interface MessageHandler { - void handleMessage(M message) throws Exception; + void handleMessage(WebSocket webSocket, M message); } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ReceiptAckSender.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ReceiptAckSender.java index a286ea2b7..4ebf7fd5f 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ReceiptAckSender.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ReceiptAckSender.java @@ -16,30 +16,43 @@ package org.dizitart.no2.sync.handlers; -import org.dizitart.no2.sync.MessageTemplate; -import org.dizitart.no2.sync.ReplicationTemplate; -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.dizitart.no2.sync.message.DataGateMessage; -import org.dizitart.no2.sync.message.Receipt; -import org.dizitart.no2.sync.message.ReceiptAware; +import okhttp3.WebSocket; +import org.dizitart.no2.sync.crdt.DeltaStates; +import org.dizitart.no2.sync.DataGateSocketListener; +import org.dizitart.no2.sync.ReplicatedCollection; +import org.dizitart.no2.sync.message.*; /** * @author Anindya Chatterjee */ public interface ReceiptAckSender { - ReplicationTemplate getReplicationTemplate(); + ReplicatedCollection getReplicatedCollection(); - Ack createAck(String correlationId, Receipt receipt); + Ack createAck(String transactionId, Receipt receipt); - default void sendAck(ReceiptAware message) { + default void sendAck(WebSocket webSocket, ReceiptAware message) { if (message != null) { - LastWriteWinState state = message.getFeed(); - getReplicationTemplate().getCrdt().merge(state); + DeltaStates state = message.getFeed(); + getReplicatedCollection().getReplicatedDataType().merge(state); Receipt receipt = message.calculateReceipt(); - Ack ack = createAck(message.getHeader().getId(), receipt); - MessageTemplate messageTemplate = getReplicationTemplate().getMessageTemplate(); - messageTemplate.sendMessage(ack); + Ack ack = createAck(message.getHeader().getTransactionId(), receipt); + ack.getHeader().setCorrelationId(message.getHeader().getId()); + + BatchMessage batchMessage = (BatchMessage) message; + BatchMessage batchAck = (BatchMessage) ack; + + // set offset and batch size + batchAck.setNextOffset(batchMessage.getNextOffset()); + batchAck.setBatchSize(batchMessage.getBatchSize()); + + // set start time and end time + batchAck.setStartMarkers(batchMessage.getStartMarkers()); + batchAck.setEndMarkers(batchMessage.getEndMarkers()); + + DataGateSocketListener dataGateSocketListener + = getReplicatedCollection().getDataGateSocketListener(); + dataGateSocketListener.sendMessage(webSocket, ack); } } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ReceiptLedgerAware.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ReceiptLedgerAware.java new file mode 100644 index 000000000..283dd371a --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/handlers/ReceiptLedgerAware.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017-2020. Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dizitart.no2.sync.handlers; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.collection.NitriteId; +import org.dizitart.no2.sync.crdt.ConflictFreeReplicatedDataType; +import org.dizitart.no2.sync.crdt.DeltaStates; +import org.dizitart.no2.sync.message.Receipt; + +import java.util.HashMap; +import java.util.HashSet; + +/** + * TODO: documentation + * @author Anindya Chatterjee + */ +public interface ReceiptLedgerAware { + + ConflictFreeReplicatedDataType getReplicatedDataType(); + + default DeltaStates createState(Receipt receipt) { + DeltaStates state = new DeltaStates(); + state.setTombstoneMap(new HashMap<>()); + state.setChangeSet(new HashSet<>()); + + if (receipt != null) { + if (receipt.getAdded() != null) { + for (String id : receipt.getAdded()) { + Document document = getReplicatedDataType().getDocument(NitriteId.createId(id)); + if (document != null) { + state.getChangeSet().add(document); + } + } + } + + if (receipt.getRemoved() != null) { + for (String id : receipt.getRemoved()) { + Long timestamp = getReplicatedDataType().getTombstoneTime(NitriteId.createId(id)); + if (timestamp != null) { + state.getTombstoneMap().put(id, timestamp); + } + } + } + } + + return state; + } + + default boolean shouldRetry(Receipt receipt) { + if (receipt == null) return false; + if (receipt.getAdded() == null) return false; + if (receipt.getAdded() == null && receipt.getRemoved() == null) return false; + return !receipt.getAdded().isEmpty() || !receipt.getRemoved().isEmpty(); + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchAck.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchAck.java index 91ba6c856..b32f9d067 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchAck.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchAck.java @@ -17,12 +17,14 @@ package org.dizitart.no2.sync.message; import lombok.Data; +import lombok.EqualsAndHashCode; /** * @author Anindya Chatterjee */ @Data -public class BatchAck implements DataGateMessage { +@EqualsAndHashCode(callSuper = true) +public class BatchAck extends BatchMessage { private MessageHeader header; private Receipt receipt; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeContinue.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeContinue.java index 0c673b657..ab7af2ebf 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeContinue.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeContinue.java @@ -17,15 +17,15 @@ package org.dizitart.no2.sync.message; import lombok.Data; -import org.dizitart.no2.sync.crdt.LastWriteWinState; +import lombok.EqualsAndHashCode; +import org.dizitart.no2.sync.crdt.DeltaStates; /** * @author Anindya Chatterjee */ @Data -public class BatchChangeContinue implements ReceiptAware { +@EqualsAndHashCode(callSuper = true) +public class BatchChangeContinue extends BatchMessage implements ReceiptAware { private MessageHeader header; - private LastWriteWinState feed; - private Integer batchSize; - private Integer debounce; + private DeltaStates feed; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeEnd.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeEnd.java index e41289990..a20f23ba1 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeEnd.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeEnd.java @@ -17,14 +17,13 @@ package org.dizitart.no2.sync.message; import lombok.Data; +import lombok.EqualsAndHashCode; /** * @author Anindya Chatterjee */ @Data -public class BatchChangeEnd implements DataGateMessage { +@EqualsAndHashCode(callSuper = true) +public class BatchChangeEnd extends BatchMessage { private MessageHeader header; - private Long lastSynced; - private Integer batchSize; - private Integer debounce; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeStart.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeStart.java index 4f80ab10a..41927a5b1 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeStart.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchChangeStart.java @@ -17,15 +17,17 @@ package org.dizitart.no2.sync.message; import lombok.Data; -import org.dizitart.no2.sync.crdt.LastWriteWinState; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.dizitart.no2.sync.crdt.DeltaStates; /** * @author Anindya Chatterjee */ @Data -public class BatchChangeStart implements ReceiptAware { +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class BatchChangeStart extends BatchMessage implements ReceiptAware { private MessageHeader header; - private Integer batchSize; - private Integer debounce; - private LastWriteWinState feed; + private DeltaStates feed; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchEndAck.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchEndAck.java index 2dc73e2a0..575e7bb1c 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchEndAck.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchEndAck.java @@ -17,11 +17,13 @@ package org.dizitart.no2.sync.message; import lombok.Data; +import lombok.EqualsAndHashCode; /** * @author Anindya Chatterjee */ @Data -public class BatchEndAck implements DataGateMessage { +@EqualsAndHashCode(callSuper = true) +public class BatchEndAck extends BatchMessage { private MessageHeader header; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchMessage.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchMessage.java new file mode 100644 index 000000000..f8ab01f0e --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/BatchMessage.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.sync.message; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; +import org.dizitart.no2.sync.crdt.Markers; + +/** + * @author Anindya Chatterjee + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class BatchMessage implements DataGateMessage { + private Integer nextOffset; + private Integer batchSize; + private Markers startMarkers; + private Markers endMarkers; +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ConnectAck.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ConnectAck.java index d974f486f..99f7f131e 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ConnectAck.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ConnectAck.java @@ -17,12 +17,14 @@ package org.dizitart.no2.sync.message; import lombok.Data; +import lombok.EqualsAndHashCode; /** * @author Anindya Chatterjee */ @Data -public class ConnectAck implements DataGateMessage { +@EqualsAndHashCode(callSuper = true) +public class ConnectAck extends BatchMessage { private MessageHeader header; private Long tombstoneTtl; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/DataGateFeed.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/DataGateFeed.java index 18e85ed07..053a3ea6b 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/DataGateFeed.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/DataGateFeed.java @@ -17,7 +17,7 @@ package org.dizitart.no2.sync.message; import lombok.Data; -import org.dizitart.no2.sync.crdt.LastWriteWinState; +import org.dizitart.no2.sync.crdt.DeltaStates; /** * @author Anindya Chatterjee. @@ -25,5 +25,5 @@ @Data public class DataGateFeed implements ReceiptAware { private MessageHeader header; - private LastWriteWinState feed; + private DeltaStates feed; } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/MessageHeader.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/MessageHeader.java index 0c4136871..3c9aaaa1b 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/MessageHeader.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/MessageHeader.java @@ -24,7 +24,9 @@ @Data public class MessageHeader { private String id; + private String transactionId; private String correlationId; + private String tenant; private String collection; private String userName; private Long timestamp; diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/Receipt.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/Receipt.java index e560ea26c..95dc45aa7 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/Receipt.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/Receipt.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.dizitart.no2.collection.Document; import java.util.HashSet; import java.util.Set; @@ -32,4 +33,17 @@ public class Receipt { private Set added = new HashSet<>(); private Set removed = new HashSet<>(); + + @SuppressWarnings("unchecked") + public static Receipt fromDocument(Document document) { + return new Receipt( + (Set) document.get("added", Set.class), + (Set) document.get("removed", Set.class) + ); + } + + public Document toDocument() { + return Document.createDocument("added", added) + .put("removed", removed); + } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ReceiptAware.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ReceiptAware.java index a3ec12480..d65f70479 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ReceiptAware.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/message/ReceiptAware.java @@ -17,7 +17,7 @@ package org.dizitart.no2.sync.message; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.sync.crdt.LastWriteWinState; +import org.dizitart.no2.sync.crdt.DeltaStates; import java.util.HashSet; import java.util.Map; @@ -27,21 +27,21 @@ * @author Anindya Chatterjee */ public interface ReceiptAware extends DataGateMessage { - LastWriteWinState getFeed(); + DeltaStates getFeed(); default Receipt calculateReceipt() { Set added = new HashSet<>(); Set removed = new HashSet<>(); if (getFeed() != null) { - if (getFeed().getChanges() != null) { - for (Document change : getFeed().getChanges()) { + if (getFeed().getChangeSet() != null) { + for (Document change : getFeed().getChangeSet()) { added.add(change.getId().getIdValue()); } } - if (getFeed().getTombstones() != null) { - for (Map.Entry entry : getFeed().getTombstones().entrySet()) { + if (getFeed().getTombstoneMap() != null) { + for (Map.Entry entry : getFeed().getTombstoneMap().entrySet()) { removed.add(entry.getKey()); } } diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/CloseReason.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/CloseReason.java new file mode 100644 index 000000000..00e2b9ace --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/CloseReason.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.sync.net; + +import lombok.Data; +import org.dizitart.no2.sync.ReplicationException; + +/** + * @author Anindya Chatterjee + */ +@Data +public class CloseReason { + public static final CloseReason ServerClose = new CloseReason("Server Disconnected"); + public static final CloseReason ClientClose = new CloseReason("User Disconnected"); + + private final String reason; + private final Throwable error; + + public CloseReason(String reason) { + this.reason = reason; + this.error = null; + } + + public CloseReason(String reason, Throwable error) { + this.reason = reason; + this.error = error; + } + + public String getReasonMessage() { + Throwable cause = error; + while (true) { + if (cause == null) { + return reason; + } else if (cause instanceof ReplicationException) { + String message = cause.getMessage(); + while (true) { + if (message.length() > 123) { + message = message.substring(0, message.lastIndexOf(':')); + } else { + return message; + } + } + } + cause = error.getCause(); + } + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateClient.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateClient.java new file mode 100644 index 000000000..1c207f532 --- /dev/null +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateClient.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.sync.net; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.WebSocketListener; +import org.dizitart.no2.sync.Config; +import org.dizitart.no2.sync.ReplicationException; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.util.concurrent.TimeUnit; + +/** + * @author Anindya Chatterjee + */ +@Slf4j +public class DataGateClient { + private final Config config; + + private OkHttpClient httpClient; + private Request request; + + public DataGateClient(Config config) { + this.config = config; + configure(config); + } + + public void setListener(WebSocketListener listener) { + try { + createWebSocket(listener); + } catch (Exception e) { + log.error("Failed to connect to remote datagate server", e); + throw new ReplicationException("Remote datagate connection failed", e, true); + } + } + + private void configure(Config config) { + this.httpClient = createClient(); + this.request = config.getRequestBuilder().build(); + } + + private OkHttpClient createClient() { + OkHttpClient.Builder builder = new OkHttpClient.Builder() + .connectTimeout(config.getTimeout().getTime(), + config.getTimeout().getTimeUnit()) + .readTimeout(config.getTimeout().getTime(), + config.getTimeout().getTimeUnit()) + .writeTimeout(config.getTimeout().getTime(), + config.getTimeout().getTimeUnit()) + .pingInterval(10, TimeUnit.SECONDS) + .retryOnConnectionFailure(false); + + if (config.getProxy() != null) { + builder.proxy(config.getProxy()); + } + + if (config.isAcceptAllCertificates()) { + try { + final TrustManager[] trustAllCerts = new TrustManager[]{ + new X509TrustManager() { + + @Override + @SuppressWarnings("TrustAllX509TrustManager") + public void checkClientTrusted(java.security.cert.X509Certificate[] chain, + String authType) { + } + + @Override + @SuppressWarnings("TrustAllX509TrustManager") + public void checkServerTrusted(java.security.cert.X509Certificate[] chain, + String authType) { + } + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new java.security.cert.X509Certificate[]{}; + } + } + }; + + // SSLContext needs to be compatible with TLS 1.2 + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); + + final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); + + builder.sslSocketFactory(sslSocketFactory, (X509TrustManager) trustAllCerts[0]); + + builder.hostnameVerifier((hostname, session) -> true); + } catch (Exception e) { + throw new ReplicationException("Error while configuring SSLSocketFactory", e, true); + } + } + + return builder.build(); + } + + private void createWebSocket(WebSocketListener webSocketListener) { + if (httpClient != null) { + httpClient.dispatcher().cancelAll(); + httpClient.newWebSocket(request, webSocketListener); + } + } +} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateSocket.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateSocket.java deleted file mode 100644 index 3aa5e9c60..000000000 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateSocket.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.sync.net; - - -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; -import okhttp3.*; -import okio.ByteString; -import org.dizitart.no2.sync.Config; -import org.dizitart.no2.sync.ReplicationException; -import org.dizitart.no2.sync.message.DataGateMessage; -import org.jetbrains.annotations.NotNull; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * @author Anindya Chatterjee - */ -@Slf4j -public class DataGateSocket { - private final static int RECONNECT_INTERVAL = 10 * 1000; - private final static long RECONNECT_MAX_TIME = 120 * 1000; - private final OkHttpClient httpClient; - private final Request request; - private final Lock lock; - private final ObjectMapper objectMapper; - private final Config config; - private final Callable networkConnectivityChecker; - private WebSocket mWebSocket; - private int currentStatus = Status.DISCONNECTED; - private boolean manualClose; - private DataGateSocketListener listener; - private int reconnectCount = 0; - private Timer reconnectTimer; - private CountDownLatch latch; - private final WebSocketListener webSocketListener = new WebSocketListener() { - @Override - public void onOpen(@NotNull WebSocket webSocket, @NotNull final Response response) { - mWebSocket = webSocket; - setCurrentStatus(Status.CONNECTED); - if (latch != null) { - latch.countDown(); - } - - connected(); - if (listener != null) { - listener.onOpen(response); - } - } - - @Override - public void onMessage(@NotNull WebSocket webSocket, @NotNull final String text) { - if (listener != null) { - listener.onMessage(text); - } - } - - @Override - public void onMessage(@NotNull WebSocket webSocket, @NotNull ByteString bytes) { - if (listener != null) { - listener.onMessage(bytes); - } - } - - @Override - public void onClosing(@NotNull WebSocket webSocket, int code, @NotNull String reason) { - if (listener != null) { - listener.onClosing(code, reason); - } - } - - @Override - public void onClosed(@NotNull WebSocket webSocket, int code, @NotNull String reason) { - if (listener != null) { - listener.onClosed(code, reason); - } - } - - @Override - public void onFailure(@NotNull WebSocket webSocket, @NotNull Throwable t, Response response) { - if (listener != null) { - listener.onFailure(t, response); - } - } - }; - - public DataGateSocket(Config config) { - this.config = config; - this.networkConnectivityChecker = config.getNetworkConnectivityChecker(); - this.lock = new ReentrantLock(); - this.httpClient = createClient(); - this.request = config.getRequestBuilder().build(); - this.objectMapper = config.getObjectMapper(); - } - - public void setListener(DataGateSocketListener listener) { - this.listener = listener; - } - - public boolean isConnected() { - return currentStatus == Status.CONNECTED; - } - - public int getCurrentStatus() { - return currentStatus; - } - - public void setCurrentStatus(int status) { - this.currentStatus = status; - } - - public void startConnect() { - manualClose = false; - buildConnect(); - } - - public void stopConnect(String reason) { - manualClose = true; - disconnect(reason); - } - - public boolean sendMessage(DataGateMessage message) { - boolean isSent = false; - try { - if (mWebSocket != null && isConnected()) { - String text = objectMapper.writeValueAsString(message); - log.debug("Sending message to server {}", text); - isSent = mWebSocket.send(text); - - if (!isSent) { - tryReconnect(); - } - } - } catch (Exception e) { - log.error("Error while sending message", e); - isSent = false; - } - return isSent; - } - - private void initWebSocket() { - if (httpClient != null) { - httpClient.dispatcher().cancelAll(); - } - try { - lock.lockInterruptibly(); - try { - latch = new CountDownLatch(1); - if (httpClient != null) { - httpClient.newWebSocket(request, webSocketListener); - } - latch.await(); - } finally { - lock.unlock(); - } - } catch (InterruptedException ignored) { - } - } - - private OkHttpClient createClient() { - OkHttpClient.Builder builder = new OkHttpClient.Builder() - .connectTimeout(config.getTimeout().getTime(), - config.getTimeout().getTimeUnit()) - .readTimeout(config.getTimeout().getTime(), - config.getTimeout().getTimeUnit()) - .writeTimeout(config.getTimeout().getTime(), - config.getTimeout().getTimeUnit()) - .pingInterval(10, TimeUnit.SECONDS) - .retryOnConnectionFailure(false); - - if (config.getProxy() != null) { - builder.proxy(config.getProxy()); - } - - if (config.isAcceptAllCertificates()) { - try { - final TrustManager[] trustAllCerts = new TrustManager[]{ - new X509TrustManager() { - - @Override - @SuppressWarnings("TrustAllX509TrustManager") - public void checkClientTrusted(java.security.cert.X509Certificate[] chain, - String authType) { - } - - @Override - @SuppressWarnings("TrustAllX509TrustManager") - public void checkServerTrusted(java.security.cert.X509Certificate[] chain, - String authType) { - } - - @Override - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return new java.security.cert.X509Certificate[]{}; - } - } - }; - - // SSLContext needs to be compatible with TLS 1.2 - final SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); - - final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); - - builder.sslSocketFactory(sslSocketFactory, (X509TrustManager) trustAllCerts[0]); - - builder.hostnameVerifier((hostname, session) -> true); - } catch (Exception e) { - throw new ReplicationException("error while configuring SSLSocketFactory", e, true); - } - } - - return builder.build(); - } - - private void tryReconnect() { - if (manualClose) { - return; - } - - if (isNetworkDisconnected()) { - setCurrentStatus(Status.DISCONNECTED); - return; - } - - setCurrentStatus(Status.RECONNECT); - reconnectTimer = new Timer(); - - long delay = (long) reconnectCount * RECONNECT_INTERVAL; - reconnectTimer.schedule(new TimerTask() { - @Override - public void run() { - if (listener != null) { - listener.onReconnect(); - } - buildConnect(); - } - }, Math.min(delay, RECONNECT_MAX_TIME)); - reconnectCount++; - } - - private void cancelReconnect() { - if (reconnectTimer != null) { - reconnectTimer.cancel(); - } - reconnectCount = 0; - } - - private void connected() { - cancelReconnect(); - } - - private void disconnect(String reason) { - if (currentStatus == Status.DISCONNECTED) { - return; - } - - cancelReconnect(); - - if (mWebSocket != null) { - boolean isClosed = mWebSocket.close(Status.CODE.NORMAL_CLOSE, reason); - if (!isClosed) { - if (listener != null) { - listener.onClosed(Status.CODE.ABNORMAL_CLOSE, reason); - } - } - } - - setCurrentStatus(Status.DISCONNECTED); - } - - private void buildConnect() { - if (isNetworkDisconnected()) { - setCurrentStatus(Status.DISCONNECTED); - return; - } - - switch (getCurrentStatus()) { - case Status.CONNECTED: - case Status.CONNECTING: - break; - default: - setCurrentStatus(Status.CONNECTING); - initWebSocket(); - } - } - - private boolean isNetworkDisconnected() { - try { - return !networkConnectivityChecker.call(); - } catch (Exception e) { - log.error("Network connectivity failed", e); - return true; - } - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/Status.java b/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/WebSocketCode.java similarity index 59% rename from nitrite-replication/src/main/java/org/dizitart/no2/sync/net/Status.java rename to nitrite-replication/src/main/java/org/dizitart/no2/sync/net/WebSocketCode.java index 16738c08e..49bf362df 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/Status.java +++ b/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/WebSocketCode.java @@ -1,17 +1,18 @@ /* - * Copyright (c) 2017-2020. Nitrite author or authors. + * Copyright (c) 2017-2021 Nitrite author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ package org.dizitart.no2.sync.net; @@ -19,14 +20,7 @@ /** * @author Anindya Chatterjee */ -public class Status { - public final static int CONNECTED = 1; - public final static int CONNECTING = 0; - public final static int RECONNECT = 2; - public final static int DISCONNECTED = -1; - - interface CODE { - int NORMAL_CLOSE = 1000; - int ABNORMAL_CLOSE = 1001; - } +public interface WebSocketCode { + int NORMAL_CLOSE = 1000; + int ABNORMAL_CLOSE = 1001; } diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/IntegrationTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/IntegrationTest.java new file mode 100644 index 000000000..05aea5389 --- /dev/null +++ b/nitrite-replication/src/test/java/org/dizitart/no2/IntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2; + +/** + * @author Anindya Chatterjee + */ +public interface IntegrationTest { +} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/Retry.java b/nitrite-replication/src/test/java/org/dizitart/no2/Retry.java similarity index 52% rename from nitrite-replication/src/test/java/org/dizitart/no2/integration/Retry.java rename to nitrite-replication/src/test/java/org/dizitart/no2/Retry.java index 71ee701da..11cd2b109 100644 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/Retry.java +++ b/nitrite-replication/src/test/java/org/dizitart/no2/Retry.java @@ -1,5 +1,23 @@ -package org.dizitart.no2.integration; +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2; +import lombok.extern.slf4j.Slf4j; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; @@ -7,6 +25,7 @@ /** * @author Anindya Chatterjee */ +@Slf4j public class Retry implements TestRule { private final int retryCount; @@ -31,10 +50,11 @@ public void evaluate() throws Throwable { return; } catch (Throwable t) { caughtThrowable = t; - System.err.println(description.getDisplayName() + ": run " + (i + 1) + " failed"); + log.info(description.getDisplayName() + ": run " + (i + 1) + " failed"); } } - System.err.println(description.getDisplayName() + ": giving up after " + retryCount + " failures"); + log.error(description.getDisplayName() + ": giving up after " + retryCount + " failures", caughtThrowable); + assert caughtThrowable != null; throw caughtThrowable; } }; diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/TestUtils.java b/nitrite-replication/src/test/java/org/dizitart/no2/TestUtils.java similarity index 61% rename from nitrite-replication/src/test/java/org/dizitart/no2/integration/TestUtils.java rename to nitrite-replication/src/test/java/org/dizitart/no2/TestUtils.java index 2091323b5..4ea16e259 100644 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/TestUtils.java +++ b/nitrite-replication/src/test/java/org/dizitart/no2/TestUtils.java @@ -1,40 +1,50 @@ /* - * Copyright (c) 2017-2020. Nitrite author or authors. + * Copyright (c) 2017-2021 Nitrite author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ -package org.dizitart.no2.integration; +package org.dizitart.no2; -import org.dizitart.no2.Nitrite; +import com.github.javafaker.Faker; +import lombok.SneakyThrows; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.collection.NitriteId; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.common.Constants; import org.dizitart.no2.mvstore.MVStoreModule; +import org.dizitart.no2.store.NitriteMap; +import org.dizitart.no2.store.NitriteStore; import org.junit.Assert; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.List; -import java.util.Random; +import java.util.UUID; import java.util.stream.Collectors; +import static org.junit.Assert.assertTrue; + /** * @author Anindya Chatterjee */ public class TestUtils { + private final static Faker faker = new Faker(); private TestUtils() {} - private static final Random random = new Random(); - public static Nitrite createDb() { MVStoreModule storeModule = MVStoreModule.withConfig() .build(); @@ -57,6 +67,11 @@ public static Nitrite createDb(String filePath) { .openOrCreate(); } + @SneakyThrows + public static void deleteDb(String filePath) { + Files.delete(Paths.get(filePath)); + } + public static void assertEquals(NitriteCollection c1, NitriteCollection c2) { List l1 = c1.find().toList().stream().map(TestUtils::trimMeta).collect(Collectors.toList()); List l2 = c2.find().toList().stream().map(TestUtils::trimMeta).collect(Collectors.toList()); @@ -74,25 +89,30 @@ public static Document trimMeta(Document document) { document.remove(Constants.DOC_REVISION); document.remove(Constants.DOC_MODIFIED); document.remove(Constants.DOC_SOURCE); + document.remove("_synced"); return document; } public static Document randomDocument() { return Document.createDocument() - .put("firstName", randomString()) - .put("lastName", randomString()) - .put("age", random.nextInt()); + .put("firstName", faker.name().firstName()) + .put("lastName", faker.name().lastName()) + .put("age", faker.random().nextLong()); } - private static String randomString() { - int leftLimit = 97; // letter 'a' - int rightLimit = 122; // letter 'z' - int targetStringLength = 10; - - return random.ints(leftLimit, rightLimit + 1) - .limit(targetStringLength) - .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) - .toString(); + public static String getRandomTempDbFile() { + String dataDir = System.getProperty("java.io.tmpdir") + File.separator + + "nitrite" + File.separator + "data"; + File file = new File(dataDir); + if (!file.exists()) { + assertTrue(file.mkdirs()); + } + return file.getPath() + File.separator + UUID.randomUUID() + ".db"; } + public static NitriteMap getTombstone(NitriteCollection collection) { + NitriteStore nitriteStore = collection.getStore(); + String tombStoneName = collection.getAttributes().get(Attributes.TOMBSTONE); + return nitriteStore.openMap(tombStoneName, NitriteId.class, Document.class); + } } diff --git a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonExtension.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/BooleanResponse.java similarity index 63% rename from nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonExtension.java rename to nitrite-replication/src/test/java/org/dizitart/no2/integration/BooleanResponse.java index d6be1d6e1..3df6c7a0c 100644 --- a/nitrite-jackson-mapper/src/main/java/org/dizitart/no2/common/mapper/JacksonExtension.java +++ b/nitrite-replication/src/test/java/org/dizitart/no2/integration/BooleanResponse.java @@ -1,29 +1,28 @@ /* - * Copyright (c) 2017-2020. Nitrite author or authors. + * Copyright (c) 2017-2021 Nitrite author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ -package org.dizitart.no2.common.mapper; - -import com.fasterxml.jackson.databind.Module; +package org.dizitart.no2.integration; -import java.util.List; +import lombok.Data; /** * @author Anindya Chatterjee */ -public interface JacksonExtension { - List> getSupportedTypes(); - Module getModule(); +@Data +public class BooleanResponse { + private Boolean result; } diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/DataGateIntegrationTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/DataGateIntegrationTest.java index d243415ac..7a8cba03e 100644 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/DataGateIntegrationTest.java +++ b/nitrite-replication/src/test/java/org/dizitart/no2/integration/DataGateIntegrationTest.java @@ -1,204 +1,821 @@ /* - * Copyright (c) 2017-2020. Nitrite author or authors. + * Copyright (c) 2017-2021 Nitrite author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ package org.dizitart.no2.integration; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; -import okhttp3.*; import org.dizitart.no2.Nitrite; +import org.dizitart.no2.Retry; +import org.dizitart.no2.TestUtils; import org.dizitart.no2.collection.Document; +import org.dizitart.no2.collection.DocumentCursor; import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.filters.Filter; import org.dizitart.no2.sync.Replica; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.dizitart.no2.collection.Document.createDocument; -import static org.dizitart.no2.integration.TestUtils.createDb; +import org.dizitart.no2.sync.event.ReplicationEvent; +import org.dizitart.no2.sync.event.ReplicationEventType; +import org.junit.*; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.shaded.org.bouncycastle.util.Objects; +import org.testcontainers.utility.DockerImageName; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.dizitart.no2.TestUtils.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.*; /** * @author Anindya Chatterjee */ @Slf4j +@Ignore public class DataGateIntegrationTest { - // TODO: Use https://www.testcontainers.org/ and use datagate-container for integration test - - public static void main(String[] args) { - Path dbPath = null; - try { - createUser(); - dbPath = Files.createTempFile("no2-datagate-it", "db"); - - Nitrite db = createDb(dbPath.toFile().getPath()); - - NitriteCollection collection = db.getCollection("datagateIntegration"); - Document document = createDocument().put("firstName", "Anindya") - .put("lastName", "Chatterjee") - .put("address", createDocument("street", "1234 Abcd Street") - .put("pin", 123456)); - collection.insert(document); - - String jwt = getToken(); - - System.out.println("Token - " + jwt); - Replica replica = Replica.builder() - .of(collection) - .remote("wss://127.0.0.1:3030/ws/datagate/abcd@gmail.com/datagateIntegration") - .jwtAuth("abcd@gmail.com", jwt) - .acceptAllCertificates(true) - .create(); - -// replica.subscribe(event -> { -// if (event.getEventType() == ReplicationEventType.Stopped) { -// System.out.println("Reconnecting"); -// replica.connect(); -// } -// }); - - replica.connect(); - System.out.println("Connected"); - Thread.sleep(10000); - System.out.println("Completed"); - System.out.println("Collection Size - " + collection.size()); - for (Document d : collection.find()) { - System.out.println(d); + private final Network network = Network.newNetwork(); + private String dbFile1, dbFile2; + private Nitrite db1, db2; + private GenericContainer datagate; + private MongoDBContainer mongodb; + private GenericContainer mongoSeed; + + @Rule(order = 0) + public Retry retry = new Retry(3); + + @Before + public void setUp() throws Exception { + Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(log); + + mongodb = new MongoDBContainer( + DockerImageName.parse("mongo:latest")) + .withNetwork(network) + .withNetworkAliases("mongo") + .withExposedPorts(27017); + + mongoSeed = new GenericContainer<>(new ImageFromDockerfile() + .withFileFromClasspath("appConfig.json", "mongo-seed/appConfig.json") + .withFileFromClasspath("serverConfig.json", "mongo-seed/serverConfig.json") + .withFileFromClasspath("Dockerfile", "mongo-seed/Dockerfile")) + .withNetwork(network); + + mongodb.start(); + mongoSeed.start(); + + datagate = new GenericContainer<>( + DockerImageName.parse("nitrite/nitrite-datagate:latest")) + .withEnv("MONGO_URL", "mongodb://mongo:27017/datagate") + .withImagePullPolicy(imageName -> false) + .withNetwork(network) + .withLogConsumer(logConsumer) + .withExposedPorts(46005); + + datagate.start(); + + UserClient.createUser(datagate.getHost(), datagate.getFirstMappedPort(), "abcd@gmail.com"); + UserClient.createUser(datagate.getHost(), datagate.getFirstMappedPort(), "abcd2@gmail.com"); + UserClient.createUser(datagate.getHost(), datagate.getFirstMappedPort(), "abcd3@gmail.com"); + } + + @After + public void cleanUp() { + mongodb.stop(); + mongoSeed.stop(); + datagate.stop(); + + if (db1 != null && dbFile1 != null) { + if (!db1.isClosed()) { + db1.close(); + } + TestUtils.deleteDb(dbFile1); + } + + if (db2 != null && dbFile2 != null) { + if (!db2.isClosed()) { + db2.close(); } - } catch (Exception e) { - e.printStackTrace(); - } finally { + TestUtils.deleteDb(dbFile2); + } + } + + @Test + public void testSingleUserSingleReplica() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt = UserClient.getToken(host, port, "abcd@gmail.com"); + + db1 = createDb(dbFile1); + NitriteCollection c1 = db1.getCollection("testSingleUserSingleReplica"); + + Replica replica = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("integration-test") + .jwtAuth("abcd@gmail.com", jwt) + .create(); + + replica.connect(); + + Random random = new Random(); + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); try { - if (dbPath != null) { - Files.delete(dbPath); - } - } catch (Exception e) { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { e.printStackTrace(); } } + + await().atMost(5, SECONDS).until(() -> c1.size() == 10); + c1.remove(Filter.ALL); + await().atMost(5, SECONDS).until(() -> c1.size() == 0); + replica.disconnectNow(); } - private static void createUser() throws Exception { - OkHttpClient client = getUnsafeOkHttpClient(); - Request request = new Request.Builder() - .url("https://127.0.0.1:3030/exists?email=abcd@gmail.com") - .build(); - - Call call = client.newCall(request); - Response response = call.execute(); - ObjectMapper mapper = new ObjectMapper(); - assert response.body() != null; - JsonNode jsonNode = mapper.readValue(response.body().string(), JsonNode.class); - if (jsonNode.has("exists")) { - if (jsonNode.get("exists").asBoolean()) { - return; + @Test + public void testSingleUserMultiReplica() throws Exception { + dbFile1 = getRandomTempDbFile(); + dbFile2 = getRandomTempDbFile(); + + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + db1 = createDb(dbFile1); + db2 = createDb(dbFile2); + + NitriteCollection c1 = db1.getCollection("testSingleUserMultiReplica"); + NitriteCollection c2 = db2.getCollection("testSingleUserMultiReplica"); + + String jwt = UserClient.getToken(host, port, "abcd@gmail.com"); + + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("integration-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r1") + .acceptAllCertificates(true) + .create(); + + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("integration-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r2") + .acceptAllCertificates(true) + .create(); + + r1.connect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + } + + await().atMost(5, SECONDS).until(() -> c1.size() == 10); + assertEquals(c2.size(), 0); + + r2.connect(); + await().atMost(15, SECONDS).until(() -> c2.size() == 10); + + Random random = new Random(); + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + try { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + for (int i = 0; i < 20; i++) { + Document document = randomDocument(); + c2.insert(document); + try { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + + await().atMost(10, SECONDS).until(() -> c1.size() == 40); + assertEquals(c2.size(), 40); + + r1.disconnect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + try { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); } } - String json = "{" + - "\"email\":\"abcd@gmail.com\"," + - "\"password\":\"chang3me\"," + - "\"firstName\":\"Anindya\"," + - "\"lastName\":\"Chatterjee\"," + - "\"roles\": [\"admin\"]}"; - RequestBody body = RequestBody.create( - MediaType.parse("application/json"), json); + for (int i = 0; i < 20; i++) { + Document document = randomDocument(); + c2.insert(document); + try { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } - request = new Request.Builder() - .url("https://127.0.0.1:3030/register") - .post(body) - .build(); + r1.connect(); + await().atMost(10, SECONDS).until(() -> c1.size() == 70 && c2.size() == 70); + TestUtils.assertEquals(c1, c2); - call = client.newCall(request); - response = call.execute(); + c2.remove(Filter.ALL); + + await().atMost(10, SECONDS).until(() -> c2.size() == 0); + + await().atMost(30, SECONDS).until(() -> c1.size() == 0); + TestUtils.assertEquals(c1, c2); + + r1.disconnectNow(); + r2.disconnectNow(); + } + + @Test + public void testMultiUserSingleReplica() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt1 = UserClient.getToken(host, port, "abcd@gmail.com"); + String jwt2 = UserClient.getToken(host, port, "abcd2@gmail.com"); + String jwt3 = UserClient.getToken(host, port, "abcd3@gmail.com"); + + Nitrite db1 = createDb(); + NitriteCollection c1 = db1.getCollection("testMultiUserSingleReplica"); + + Nitrite db2 = createDb(); + NitriteCollection c2 = db2.getCollection("testMultiUserSingleReplica"); + + Nitrite db3 = createDb(); + NitriteCollection c3 = db3.getCollection("testMultiUserSingleReplica"); + + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt1) + .create(); + r1.connect(); + + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd2@gmail.com", jwt2) + .create(); + r2.connect(); + + Replica r3 = Replica.builder() + .database(db3) + .of(c3) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd3@gmail.com", jwt3) + .create(); + r3.connect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + } + + for (int i = 0; i < 20; i++) { + Document document = randomDocument(); + c2.insert(document); + } - if (response.code() != 201) { - throw new Exception("user creation failed"); + for (int i = 0; i < 30; i++) { + Document document = randomDocument(); + c3.insert(document); } + + await().atMost(5, SECONDS).until(() -> c1.size() == 10 && c2.size() == 20 && c3.size() == 30); + + TestUtils.assertNotEquals(c1, c2); + TestUtils.assertNotEquals(c1, c3); + TestUtils.assertNotEquals(c2, c3); + + r1.disconnectNow(); + r2.disconnectNow(); + r3.disconnectNow(); } - private static String getToken() throws Exception { - OkHttpClient client = getUnsafeOkHttpClient(); - String json = "{" + - "\"email\":\"abcd@gmail.com\"," + - "\"password\":\"chang3me\"}"; - RequestBody body = RequestBody.create( - MediaType.parse("application/json"), json); - - Request request = new Request.Builder() - .url("https://127.0.0.1:3030/login") - .post(body) - .build(); - - Call call = client.newCall(request); - Response response = call.execute(); - - if (response.code() == 200) { - assert response.body() != null; - json = response.body().string(); - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readValue(json, JsonNode.class); - - if (jsonNode.has("token")) { - return jsonNode.get("token").asText(); - } + @Test + public void testMultiUserMultiReplica() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt1 = UserClient.getToken(host, port, "abcd@gmail.com"); + String jwt2 = UserClient.getToken(host, port, "abcd2@gmail.com"); + + Nitrite db1 = createDb(); + NitriteCollection c1 = db1.getCollection("testMultiUserSingleReplica1"); + + Nitrite db2 = createDb(); + NitriteCollection c2 = db2.getCollection("testMultiUserSingleReplica2"); + + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt1) + .create(); + r1.connect(); + + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd2@gmail.com", jwt2) + .create(); + r2.connect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); } - throw new Exception("failed to login"); + for (int i = 0; i < 20; i++) { + Document document = randomDocument(); + c2.insert(document); + } + + await().atMost(5, SECONDS).until(() -> c1.size() == 10 && c2.size() == 20); + + TestUtils.assertNotEquals(c1, c2); + r1.disconnectNow(); + r2.disconnectNow(); } - private static OkHttpClient getUnsafeOkHttpClient() { - try { - final TrustManager[] trustAllCerts = new TrustManager[]{ - new X509TrustManager() { - - @Override - public void checkClientTrusted(java.security.cert.X509Certificate[] chain, - String authType) { - } - - @Override - public void checkServerTrusted(java.security.cert.X509Certificate[] chain, - String authType) { - } - - @Override - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return new java.security.cert.X509Certificate[]{}; - } + @Test + public void testSecurityInCorrectCredentials() { + dbFile1 = getRandomTempDbFile(); + + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + db1 = createDb(dbFile1); + NitriteCollection c1 = db1.getCollection("testSecurityInCorrectCredentials"); + + AtomicReference errorEvent = new AtomicReference<>(); + + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", "wrong_token") + .addReplicationEventListener(event -> { + if (event.getEventType() == ReplicationEventType.Error && errorEvent.get() == null) { + errorEvent.set(event); } - }; + }) + .create(); + r1.connect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + } + + assertEquals(c1.size(), 10); - final SSLContext sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); + await().atMost(5, SECONDS).until(() -> { + ReplicationEvent replicationEvent = errorEvent.get(); + return replicationEvent.getError().getMessage().contains("failed to validate token"); + }); + r1.disconnectNow(); + } - final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); + @Test + public void testCloseDbAndReconnect() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt = UserClient.getToken(host, port, "abcd@gmail.com"); + + dbFile1 = getRandomTempDbFile(); + dbFile2 = getRandomTempDbFile(); + + db1 = createDb(dbFile1); + db2 = createDb(dbFile2); + + NitriteCollection c1 = db1.getCollection("testCloseDbAndReconnect"); + NitriteCollection c2 = db2.getCollection("testCloseDbAndReconnect"); + + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r1") + .create(); + + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r2") + .create(); + + r1.connect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + } - OkHttpClient.Builder builder = new OkHttpClient.Builder(); - builder.sslSocketFactory(sslSocketFactory, (X509TrustManager) trustAllCerts[0]); + NitriteCollection finalC1 = c1; + await().atMost(5, SECONDS).until(() -> finalC1.size() == 10); + assertEquals(c2.size(), 0); - builder.hostnameVerifier((hostname, session) -> true); + r2.connect(); + await().atMost(5, SECONDS).until(() -> c2.size() == 10); - return builder.build(); - } catch (Exception e) { - throw new RuntimeException(e); + Random random = new Random(); + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); } + + for (int i = 0; i < 20; i++) { + Document document = randomDocument(); + c2.insert(document); + try { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + NitriteCollection finalC2 = c1; + await().atMost(10, SECONDS).until(() -> finalC2.size() == 40); + assertEquals(c2.size(), 40); + + r1.disconnect(); + r1.close(); + db1.close(); + + db1 = createDb(dbFile1); + c1 = db1.getCollection("testCloseDbAndReconnect"); + r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r1") + .create(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + try { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + for (int i = 0; i < 20; i++) { + Document document = randomDocument(); + c2.insert(document); + try { + Thread.sleep(random.nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + r1.connect(); + NitriteCollection finalC = c1; + await().atMost(10, SECONDS).until(() -> finalC.size() == 70 && c2.size() == 70); + TestUtils.assertEquals(c1, c2); + + c2.remove(Filter.ALL); + + await().atMost(10, SECONDS).until(() -> finalC.size() == 0); + TestUtils.assertEquals(c1, c2); + + r1.disconnectNow(); + r2.disconnectNow(); + } + + @Test + public void testDelayedConnect() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt = UserClient.getToken(host, port, "abcd@gmail.com"); + + dbFile1 = getRandomTempDbFile(); + + db1 = createDb(dbFile1); + + NitriteCollection c1 = db1.getCollection("testDelayedConnect"); + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .create(); + + r1.connect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + } + + // allow it to reach the datagate server + Thread.sleep(5000); + + r1.disconnect(); + r1.close(); + db1.close(); + + Nitrite db2 = createDb(); + NitriteCollection c2 = db2.getCollection("testDelayedConnect"); + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .create(); + r2.connect(); + await().atMost(5, SECONDS).until(() -> c2.size() == 10); + + r1.disconnectNow(); + r2.disconnectNow(); + } + + @Test + public void testDelayedConnectRemoveAll() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt = UserClient.getToken(host, port, "abcd@gmail.com"); + + dbFile1 = getRandomTempDbFile(); + + db1 = createDb(dbFile1); + NitriteCollection c1 = db1.getCollection("testDelayedConnect"); + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r1") + .create(); + + r1.connect(); + + for (int i = 0; i < 10; i++) { + Document document = randomDocument(); + c1.insert(document); + } + + c1.remove(Filter.ALL); + assertEquals(c1.size(), 0); + + r1.disconnect(); + r1.close(); + db1.close(); + + Nitrite db2 = createDb(); + NitriteCollection c2 = db2.getCollection("testDelayedConnect"); + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r2") + .create(); + + r2.connect(); + + for (int i = 0; i < 5; i++) { + Document document = randomDocument(); + c2.insert(document); + } + + db1 = createDb(dbFile1); + NitriteCollection c3 = db1.getCollection("testDelayedConnect"); + r1 = Replica.builder() + .database(db1) + .of(c3) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r1") + .create(); + + r1.connect(); + + await().atMost(5, SECONDS).until(() -> { + List l1 = c3.find().toList().stream().map(TestUtils::trimMeta).collect(Collectors.toList()); + List l2 = c2.find().toList().stream().map(TestUtils::trimMeta).collect(Collectors.toList()); + return l1.equals(l2); + }); + + r1.disconnectNow(); + r2.disconnectNow(); + } + + @Test + public void testHighestRevisionWin() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt = UserClient.getToken(host, port, "abcd@gmail.com"); + + dbFile1 = getRandomTempDbFile(); + dbFile2 = getRandomTempDbFile(); + + db1 = createDb(dbFile1); + db2 = createDb(dbFile2); + + NitriteCollection c1 = db1.getCollection("testHighestRevisionWin"); + NitriteCollection c2 = db2.getCollection("testHighestRevisionWin"); + + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r1") + .create(); + + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r2") + .create(); + + r1.connect(); + r2.connect(); + + Document document = randomDocument(); + c1.insert(document); + + await().atMost(5, SECONDS).until(() -> c2.size() == 1); + + Document d2 = c2.find().firstOrNull(); + d2.put("age", 38); + + assertTrue(Objects.areEqual(c1.find().firstOrNull().getRevision(), 1)); + assertNotEquals(c1.find().firstOrNull().get("age"), d2.get("age")); + + c2.update(d2); + + await().atMost(5, SECONDS).until(() -> { + Document d1 = c1.find().firstOrNull(); + return Objects.areEqual(d1.get("age"), 38) && Objects.areEqual(d1.getRevision(), 2); + }); + } + + @Test + public void testRandomInsertDelete() throws Exception { + String host = datagate.getHost(); + Integer port = datagate.getMappedPort(46005); + + String jwt = UserClient.getToken(host, port, "abcd@gmail.com"); + + dbFile1 = getRandomTempDbFile(); + dbFile2 = getRandomTempDbFile(); + + db1 = createDb(dbFile1); + db2 = createDb(dbFile2); + + NitriteCollection c1 = db1.getCollection("testRandomInsertDelete"); + NitriteCollection c2 = db2.getCollection("testRandomInsertDelete"); + + Replica r1 = Replica.builder() + .database(db1) + .of(c1) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r1") + .create(); + + Replica r2 = Replica.builder() + .database(db2) + .of(c2) + .remoteHost(host) + .remotePort(port) + .tenant("junit-test") + .jwtAuth("abcd@gmail.com", jwt) + .replicaName("r2") + .create(); + + r1.connect(); + r2.connect(); + + Random random = new Random(); + + // insert + for (int i = 0; i < random.nextInt(50); i++) { + Document document = randomDocument(); + c1.insert(document); + Thread.sleep(random.nextInt(100)); + } + + // insert + for (int i = 0; i < random.nextInt(30); i++) { + Document document = randomDocument(); + c2.insert(document); + Thread.sleep(random.nextInt(100)); + } + + // update + DocumentCursor cursor = c1.find(); + for (Document document : cursor) { + document.put("age", random.nextInt()); + c1.update(document); + } + + // delete + for (int i = 0; i < random.nextInt(30); i++) { + Document document = c1.find().firstOrNull(); + c1.remove(document); + Thread.sleep(random.nextInt(100)); + } + + await().atMost(20, SECONDS).until(() -> c1.size() > 0 && c1.size() == c2.size()); + System.out.println("C1 Size = " + c1.size()); + TestUtils.assertEquals(c1, c2); } } diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/DataGateResponse.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/DataGateResponse.java new file mode 100644 index 000000000..b2f4b3b00 --- /dev/null +++ b/nitrite-replication/src/test/java/org/dizitart/no2/integration/DataGateResponse.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +/** + * @author Anindya Chatterjee + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataGateResponse { + private T data; + private Integer code; + private String message; +} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/ReplicaNegativeTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/ReplicaNegativeTest.java deleted file mode 100644 index 50e7c89e8..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/ReplicaNegativeTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.integration; - -import org.dizitart.no2.Nitrite; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.common.concurrent.ThreadPoolManager; -import org.dizitart.no2.sync.Replica; -import org.dizitart.no2.sync.crdt.LastWriteWinMap; -import org.dizitart.no2.integration.server.Repository; -import org.dizitart.no2.integration.server.SimpleDataGateServer; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.util.concurrent.ExecutorService; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.awaitility.Awaitility.await; -import static org.dizitart.no2.integration.ReplicaTest.getRandomTempDbFile; -import static org.dizitart.no2.integration.TestUtils.createDb; -import static org.dizitart.no2.integration.TestUtils.randomDocument; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Anindya Chatterjee - */ -public class ReplicaNegativeTest { - private SimpleDataGateServer server; - private String dbFile; - private ExecutorService executorService; - private Repository repository; - - @Rule - public Retry retry = new Retry(3); - - @Before - public void setUp() throws Exception { - dbFile = getRandomTempDbFile(); - server = new SimpleDataGateServer(9090); - executorService = ThreadPoolManager.getThreadPool(2, "ReplicaNegativeTest"); - server.start(); - repository = Repository.getInstance(); - } - - @After - public void cleanUp() { - if (executorService != null) { - executorService.shutdown(); - } - server.stop(); - } - - @Test - public void testServerClose() { - repository.getUserMap().put("anidotnet", "abcd"); - - Nitrite db1 = createDb(dbFile); - - NitriteCollection c1 = db1.getCollection("testServerClose"); - - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testServerClose") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - - executorService.submit(() -> { - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - }); - - await().atMost(5, SECONDS).until(() -> repository.getCollectionReplicaMap().size() == 1); - assertEquals(repository.getUserReplicaMap().size(), 1); - assertTrue(repository.getUserReplicaMap().containsKey("anidotnet")); - assertTrue(repository.getCollectionReplicaMap().containsKey("anidotnet@testServerClose")); - LastWriteWinMap lastWriteWinMap = repository.getReplicaStore().get("anidotnet@testServerClose"); - - await().atMost(5, SECONDS).until(() -> lastWriteWinMap.getCollection().find().size() == 10); - server.stop(); - await().atMost(5, SECONDS).until(() -> !r1.isConnected()); - } - - /* - * 1. Server close and again restarted - * 2. Connectivity check with with atomic counter - * */ -} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/ReplicaTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/ReplicaTest.java deleted file mode 100644 index 4e1e33252..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/ReplicaTest.java +++ /dev/null @@ -1,642 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.integration; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.Nitrite; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.filters.Filter; -import org.dizitart.no2.integration.server.Repository; -import org.dizitart.no2.integration.server.SimpleDataGateServer; -import org.dizitart.no2.sync.Replica; -import org.dizitart.no2.sync.ReplicationTemplate; -import org.dizitart.no2.sync.crdt.LastWriteWinMap; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.lang.reflect.Field; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.awaitility.Awaitility.await; -import static org.dizitart.no2.collection.Document.createDocument; -import static org.dizitart.no2.common.util.DocumentUtils.isSimilar; -import static org.dizitart.no2.filters.FluentFilter.where; -import static org.dizitart.no2.integration.TestUtils.createDb; -import static org.dizitart.no2.integration.TestUtils.randomDocument; -import static org.junit.Assert.*; - -/** - * @author Anindya Chatterjee - */ -@Slf4j -public class ReplicaTest { - private static SimpleDataGateServer server; - @Rule - public Retry retry = new Retry(3); - private String dbFile; - private ExecutorService executorService; - private Repository repository; - private Nitrite db; - - public static String getRandomTempDbFile() { - String dataDir = System.getProperty("java.io.tmpdir") + File.separator + "nitrite" + File.separator + "data"; - File file = new File(dataDir); - if (!file.exists()) { - assertTrue(file.mkdirs()); - } - return file.getPath() + File.separator + UUID.randomUUID().toString() + ".db"; - } - - @Before - public void setUp() throws Exception { - server = new SimpleDataGateServer(9090); - server.start(); - dbFile = getRandomTempDbFile(); - executorService = Executors.newCachedThreadPool(); - repository = Repository.getInstance(); - } - - @After - public void cleanUp() throws Exception { - executorService.awaitTermination(2, SECONDS); - executorService.shutdown(); - - if (db != null && !db.isClosed()) { - db.close(); - } - - if (Files.exists(Paths.get(dbFile))) { - Files.delete(Paths.get(dbFile)); - } - server.stop(); - } - - @Test - public void testSingleUserSingleReplica() { - repository.getUserMap().put("anidotnet", "abcd"); - - db = createDb(dbFile); - NitriteCollection collection = db.getCollection("testSingleUserSingleReplica"); - Document document = createDocument() - .put("firstName", "Anindya") - .put("lastName", "Chatterjee") - .put("address", createDocument("street", "1234 Abcd Street") - .put("pin", 123456)); - collection.insert(document); - - Replica replica = Replica.builder() - .of(collection) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testSingleUserSingleReplica") - .jwtAuth("anidotnet", "abcd") - .create(); - - replica.connect(); - - await().atMost(5, SECONDS).until(() -> repository.getCollectionReplicaMap().size() == 1); - assertEquals(repository.getUserReplicaMap().size(), 1); - assertTrue(repository.getUserReplicaMap().containsKey("anidotnet")); - assertTrue(repository.getCollectionReplicaMap().containsKey("anidotnet@testSingleUserSingleReplica")); - LastWriteWinMap lastWriteWinMap = repository.getReplicaStore().get("anidotnet@testSingleUserSingleReplica"); - - await().atMost(5, SECONDS).until(() -> lastWriteWinMap.getCollection().find().size() == 1); - Document doc = lastWriteWinMap.getCollection().find(where("firstName").eq("Anindya")).firstOrNull(); - - assertTrue(isSimilar(document, doc, "firstName", "lastName", "address", "pin")); - - collection.remove(doc); - await().atMost(5, SECONDS).until(() -> lastWriteWinMap.getCollection().size() == 0); - doc = lastWriteWinMap.getCollection().find(where("firstName").eq("Anindya")).firstOrNull(); - assertNull(doc); - assertEquals(collection.size(), 0); - - collection.insert(document); - await().atMost(5, SECONDS).until(() -> lastWriteWinMap.getCollection().size() == 1); - doc = lastWriteWinMap.getCollection().find(where("firstName").eq("Anindya")).firstOrNull(); - assertTrue(isSimilar(document, doc, "firstName", "lastName", "address", "pin")); - - replica.disconnect(); - collection.remove(doc); - await().atMost(5, SECONDS).until(() -> lastWriteWinMap.getCollection().size() == 1); - doc = lastWriteWinMap.getCollection().find(where("firstName").eq("Anindya")).firstOrNull(); - assertTrue(isSimilar(document, doc, "firstName", "lastName", "address", "pin")); - - replica.connect(); - await().atMost(5, SECONDS).until(() -> lastWriteWinMap.getCollection().size() == 0); - doc = lastWriteWinMap.getCollection().find(where("firstName").eq("Anindya")).firstOrNull(); - assertNull(doc); - } - - @Test - public void testSingleUserMultiReplica() { - repository.getUserMap().put("anidotnet", "abcd"); - - db = createDb(dbFile); - - Nitrite db2 = createDb(); - - NitriteCollection c1 = db.getCollection("testSingleUserMultiReplica"); - NitriteCollection c2 = db2.getCollection("testSingleUserMultiReplica"); - - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testSingleUserMultiReplica") - .jwtAuth("anidotnet", "abcd") - .create(); - - Replica r2 = Replica.builder() - .of(c2) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testSingleUserMultiReplica") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - - executorService.submit(() -> { - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - }); - - await().atMost(5, SECONDS).until(() -> c1.size() == 10); - assertEquals(c2.size(), 0); - - r2.connect(); - await().atMost(5, SECONDS).until(() -> c2.size() == 10); - - Random random = new Random(); - executorService.submit(() -> { - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - - executorService.submit(() -> { - for (int i = 0; i < 20; i++) { - Document document = randomDocument(); - c2.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - - await().atMost(10, SECONDS).until(() -> c1.size() == 40); - assertEquals(c2.size(), 40); - - r1.disconnect(); - - executorService.submit(() -> { - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - - executorService.submit(() -> { - for (int i = 0; i < 20; i++) { - Document document = randomDocument(); - c2.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - - r1.connect(); - await().atMost(10, SECONDS).until(() -> c1.size() == 70 && c2.size() == 70); - TestUtils.assertEquals(c1, c2); - - executorService.submit(() -> { - c2.remove(Filter.ALL); - }); - - await().atMost(10, SECONDS).until(() -> c2.size() == 0); - await().atMost(5, SECONDS).until(() -> c1.size() == 0); - TestUtils.assertEquals(c1, c2); - } - - @Test - public void testMultiUserSingleReplica() { - repository.getUserMap().put("user1", "abcd"); - repository.getUserMap().put("user2", "abcd"); - repository.getUserMap().put("user3", "abcd"); - - Nitrite db1 = createDb(); - NitriteCollection c1 = db1.getCollection("testMultiUserSingleReplica"); - - Nitrite db2 = createDb(); - NitriteCollection c2 = db2.getCollection("testMultiUserSingleReplica"); - - Nitrite db3 = createDb(); - NitriteCollection c3 = db3.getCollection("testMultiUserSingleReplica"); - - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/user1/testSingleUserSingleReplica") - .jwtAuth("user1", "abcd") - .create(); - r1.connect(); - - Replica r2 = Replica.builder() - .of(c2) - .remote("ws://127.0.0.1:9090/datagate/user2/testSingleUserSingleReplica") - .jwtAuth("user2", "abcd") - .create(); - r2.connect(); - - Replica r3 = Replica.builder() - .of(c3) - .remote("ws://127.0.0.1:9090/datagate/user3/testSingleUserSingleReplica") - .jwtAuth("user3", "abcd") - .create(); - r3.connect(); - - executorService.submit(() -> { - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - }); - - executorService.submit(() -> { - for (int i = 0; i < 20; i++) { - Document document = randomDocument(); - c2.insert(document); - } - }); - - executorService.submit(() -> { - for (int i = 0; i < 30; i++) { - Document document = randomDocument(); - c3.insert(document); - } - }); - - await().atMost(5, SECONDS).until(() -> c1.size() == 10 && c2.size() == 20 && c3.size() == 30); - - TestUtils.assertNotEquals(c1, c2); - TestUtils.assertNotEquals(c1, c3); - TestUtils.assertNotEquals(c2, c3); - } - - @Test - public void testMultiUserMultiReplica() { - repository.getUserMap().put("user1", "abcd"); - repository.getUserMap().put("user2", "abcd"); - - Nitrite db1 = createDb(); - NitriteCollection c1 = db1.getCollection("testMultiUserSingleReplica1"); - - Nitrite db2 = createDb(); - NitriteCollection c2 = db2.getCollection("testMultiUserSingleReplica2"); - - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/user1/testMultiUserSingleReplica1") - .jwtAuth("user1", "abcd") - .create(); - r1.connect(); - - Replica r2 = Replica.builder() - .of(c2) - .remote("ws://127.0.0.1:9090/datagate/user2/testMultiUserSingleReplica2") - .jwtAuth("user2", "abcd") - .create(); - r2.connect(); - - executorService.submit(() -> { - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - }); - - executorService.submit(() -> { - for (int i = 0; i < 20; i++) { - Document document = randomDocument(); - c2.insert(document); - } - }); - - await().atMost(5, SECONDS).until(() -> c1.size() == 10 && c2.size() == 20); - - TestUtils.assertNotEquals(c1, c2); - } - - @Test - public void testSecurityInCorrectCredentials() { - repository.getUserMap().put("user", "abcd"); - - Nitrite db1 = createDb(); - NitriteCollection c1 = db1.getCollection("testSecurity"); - - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/user/testSecurity") - .jwtAuth("user", "wrong_token") - .create(); - r1.connect(); - - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - - assertEquals(c1.size(), 10); - await().atMost(5, SECONDS).until(() -> !r1.isConnected()); - } - - @Test - public void testCloseDbAndReconnect() { - repository.getUserMap().put("anidotnet", "abcd"); - - db = createDb(dbFile); - - Nitrite db2 = createDb(); - - NitriteCollection c1 = db.getCollection("testCloseDbAndReconnect"); - NitriteCollection c2 = db2.getCollection("testCloseDbAndReconnect"); - - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testCloseDbAndReconnect") - .jwtAuth("anidotnet", "abcd") - .create(); - - Replica r2 = Replica.builder() - .of(c2) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testCloseDbAndReconnect") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - - NitriteCollection finalC1 = c1; - await().atMost(5, SECONDS).until(() -> finalC1.size() == 10); - assertEquals(c2.size(), 0); - - r2.connect(); - await().atMost(5, SECONDS).until(() -> c2.size() == 10); - - Random random = new Random(); - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - executorService.submit(() -> { - for (int i = 0; i < 20; i++) { - Document document = randomDocument(); - c2.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - - NitriteCollection finalC2 = c1; - await().atMost(10, SECONDS).until(() -> finalC2.size() == 40); - assertEquals(c2.size(), 40); - - r1.disconnect(); - r1.close(); - db.close(); - - db = createDb(dbFile); - c1 = db.getCollection("testCloseDbAndReconnect"); - r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testCloseDbAndReconnect") - .jwtAuth("anidotnet", "abcd") - .create(); - - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - executorService.submit(() -> { - for (int i = 0; i < 20; i++) { - Document document = randomDocument(); - c2.insert(document); - try { - Thread.sleep(random.nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - - r1.connect(); - NitriteCollection finalC = c1; - await().atMost(10, SECONDS).until(() -> finalC.size() == 70 && c2.size() == 70); - TestUtils.assertEquals(c1, c2); - - executorService.submit(() -> { - c2.remove(Filter.ALL); - }); - - await().atMost(10, SECONDS).until(() -> c2.size() == 0); - await().atMost(5, SECONDS).until(() -> finalC.size() == 0); - TestUtils.assertEquals(c1, c2); - } - - @Test - public void testDelayedConnect() { - repository.getUserMap().put("anidotnet", "abcd"); - - Nitrite db1 = createDb(dbFile); - NitriteCollection c1 = db1.getCollection("testDelayedConnect"); - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testDelayedConnect") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - await().atMost(5, SECONDS).until(() -> c1.size() == 10); - - r1.disconnect(); - r1.close(); - db1.close(); - - Nitrite db2 = createDb(); - NitriteCollection c2 = db2.getCollection("testDelayedConnect"); - Replica r2 = Replica.builder() - .of(c2) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testDelayedConnect") - .jwtAuth("anidotnet", "abcd") - .create(); - r2.connect(); - await().atMost(5, SECONDS).until(() -> c2.size() == 10); - } - - @Test - public void testDelayedConnectRemoveAll() { - repository.getUserMap().put("anidotnet", "abcd"); - - db = createDb(dbFile); - NitriteCollection c1 = db.getCollection("testDelayedConnect"); - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testDelayedConnect") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - await().atMost(5, SECONDS).until(() -> c1.size() == 10); - c1.remove(Filter.ALL); - assertEquals(c1.size(), 0); - - r1.disconnect(); - r1.close(); - db.close(); - - Nitrite db2 = createDb(); - NitriteCollection c2 = db2.getCollection("testDelayedConnect"); - Replica r2 = Replica.builder() - .of(c2) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testDelayedConnect") - .jwtAuth("anidotnet", "abcd") - .create(); - r2.connect(); - - for (int i = 0; i < 5; i++) { - Document document = randomDocument(); - c2.insert(document); - } - - db = createDb(dbFile); - NitriteCollection c3 = db.getCollection("testDelayedConnect"); - r1 = Replica.builder() - .of(c3) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testDelayedConnect") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - await().atMost(5, SECONDS).until(() -> c3.size() == 5 && c2.size() == 5); - TestUtils.assertEquals(c3, c2); - LastWriteWinMap lastWriteWinMap = repository.getReplicaStore().get("anidotnet@testDelayedConnect"); - assertEquals(lastWriteWinMap.getTombstones().size(), 10); - } - - @Test - public void testGarbageCollect() throws InterruptedException { - repository.getUserMap().put("anidotnet", "abcd"); - db = createDb(dbFile); - NitriteCollection c1 = db.getCollection("testGarbageCollect"); - Replica r1 = Replica.builder() - .of(c1) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testGarbageCollect") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - - for (int i = 0; i < 10; i++) { - Document document = randomDocument(); - c1.insert(document); - } - await().atMost(5, SECONDS).until(() -> c1.size() == 10); - c1.remove(Filter.ALL); - assertEquals(c1.size(), 0); - - r1.disconnect(); - r1.close(); - db.close(); - - repository.setGcTtl(1L); - - db = createDb(dbFile); - NitriteCollection c2 = db.getCollection("testGarbageCollect"); - r1 = Replica.builder() - .of(c2) - .remote("ws://127.0.0.1:9090/datagate/anidotnet/testGarbageCollect") - .jwtAuth("anidotnet", "abcd") - .create(); - - r1.connect(); - - LastWriteWinMap lastWriteWinMap = getCrdt(r1); - - await().atMost(5, SECONDS).until(() -> c2.size() == 0 - && lastWriteWinMap.getTombstones().size() == 0); - } - - @SneakyThrows - private LastWriteWinMap getCrdt(Replica replica) { - Field field = Replica.class.getDeclaredField("replicationTemplate"); - field.setAccessible(true); - ReplicationTemplate replicationTemplate = (ReplicationTemplate) field.get(replica); - return replicationTemplate.getCrdt(); - } -} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/Token.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/Token.java new file mode 100644 index 000000000..5b1e222ab --- /dev/null +++ b/nitrite-replication/src/test/java/org/dizitart/no2/integration/Token.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration; + +import lombok.Data; + +/** + * @author Anindya Chatterjee + */ +@Data +public class Token { + private String token; + private Long expiresAt; +} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/UserClient.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/UserClient.java new file mode 100644 index 000000000..4dc305d9e --- /dev/null +++ b/nitrite-replication/src/test/java/org/dizitart/no2/integration/UserClient.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; + +/** + * @author Anindya Chatterjee + */ +@Slf4j +public class UserClient { + public static void createUser(String host, Integer port, String user) throws Exception { + OkHttpClient client = getUnsafeOkHttpClient(); + Request request = new Request.Builder() + .url("http://" + host + ":" + port + "/exists?email=" + user) + .build(); + + Call call = client.newCall(request); + Response response; + try { + response = call.execute(); + ObjectMapper mapper = new ObjectMapper(); + assert response.body() != null; + DataGateResponse dataGateResponse = mapper.readValue(response.body().string(), + new TypeReference>() {}); + if (dataGateResponse.getData().getResult()) { + return; + } + } catch (Exception e) { + log.error("Error checking user " + user, e); + return; + } + + String json = "{" + + "\"email\":\"" + user + "\"," + + "\"password\":\"chang3me\"," + + "\"firstName\":\"Anindya\"," + + "\"lastName\":\"Chatterjee\"," + + "\"roles\": [\"1\"]" + + "}"; + + RequestBody body = RequestBody.create( + MediaType.parse("application/json"), json); + + request = new Request.Builder() + .url("http://" + host + ":" + port + "/register") + .post(body) + .build(); + + call = client.newCall(request); + response = call.execute(); + + if (response.code() != 201) { + throw new Exception("User creation failed"); + } + } + + public static String getToken(String host, Integer port, String user) throws Exception { + OkHttpClient client = getUnsafeOkHttpClient(); + String json = "{" + + "\"email\":\"" + user + "\"," + + "\"password\":\"chang3me\"}"; + + + RequestBody body = RequestBody.create( + MediaType.parse("application/json"), json); + + Request request = new Request.Builder() + .url("http://" + host + ":" + port + "/login") + .post(body) + .build(); + + Call call = client.newCall(request); + Response response = call.execute(); + + if (response.code() == 200) { + assert response.body() != null; + json = response.body().string(); + ObjectMapper mapper = new ObjectMapper(); + DataGateResponse dataGateResponse = mapper.readValue(json, + new TypeReference>() {}); + + return dataGateResponse.getData().getToken(); + } + + throw new Exception("Failed to login"); + } + + private static OkHttpClient getUnsafeOkHttpClient() { + try { + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.retryOnConnectionFailure(true); + + return builder.build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/Repository.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/Repository.java deleted file mode 100644 index bfc433ce9..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/Repository.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.integration.server; - -import jakarta.websocket.Session; -import lombok.Data; -import org.dizitart.no2.Nitrite; -import org.dizitart.no2.sync.crdt.LastWriteWinMap; - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -import static org.dizitart.no2.integration.TestUtils.createDb; - -/** - * @author Anindya Chatterjee - */ -@Data -public class Repository { - private static Repository instance = new Repository(); - private Map> collectionReplicaMap; - private Map> userReplicaMap; - private Map replicaStore; - private Nitrite db; - private String serverId; - private Set authorizedSessions; - private Map userMap; - private Long gcTtl; - - private Repository() { - reset(); - } - - public static Repository getInstance() { - return instance; - } - - public void reset() { - collectionReplicaMap = new ConcurrentHashMap<>(); - userReplicaMap = new ConcurrentHashMap<>(); - replicaStore = new ConcurrentHashMap<>(); - authorizedSessions = new HashSet<>(); - userMap = new ConcurrentHashMap<>(); - - db = createDb(); - serverId = UUID.randomUUID().toString(); - gcTtl = 0L; - } -} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/SimpleDataGateEndpoint.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/SimpleDataGateEndpoint.java deleted file mode 100644 index d83bc8a28..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/SimpleDataGateEndpoint.java +++ /dev/null @@ -1,440 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.integration.server; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.websocket.*; -import jakarta.websocket.server.PathParam; -import jakarta.websocket.server.ServerEndpoint; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.store.NitriteMap; -import org.dizitart.no2.sync.MessageFactory; -import org.dizitart.no2.sync.MessageTransformer; -import org.dizitart.no2.sync.ReplicationException; -import org.dizitart.no2.sync.crdt.LastWriteWinMap; -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.dizitart.no2.sync.message.*; -import java.io.IOException; -import java.util.*; - -/** - * @author Anindya Chatterjee - */ -@Slf4j -@Data -@ServerEndpoint(value = "/datagate/{user}/{collection}") -public class SimpleDataGateEndpoint { - private ObjectMapper objectMapper; - private Repository repository; - private MessageFactory factory; - private MessageTransformer transformer; - - public SimpleDataGateEndpoint() { - objectMapper = new ObjectMapper(); - repository = Repository.getInstance(); - factory = new MessageFactory(); - transformer = new MessageTransformer(objectMapper); - } - - @OnOpen - public void onOpen(@PathParam("user") String user, - @PathParam("collection") String collection, - Session session) { - log.info("DataGate server connection established"); - session.getUserProperties().put("collection", user + "@" + collection); - session.getUserProperties().put("authorized", false); - } - - @OnClose - public void onClose(CloseReason reason, Session session) { - log.warn("DataGate server closed due to {}", reason.getReasonPhrase()); - repository.getAuthorizedSessions().remove(session); - } - - @OnMessage - public void onMessage(String message, Session session) { - try { - log.info("Message received at server {}", message); - DataGateMessage dataGateMessage = transformer.transform(message); - if (dataGateMessage instanceof Connect) { - Connect connect = (Connect) dataGateMessage; - handleConnect(session, connect); - } else if (dataGateMessage instanceof BatchChangeStart) { - BatchChangeStart batchChangeStart = (BatchChangeStart) dataGateMessage; - handleBatchChangeStart(session, batchChangeStart); - } else if (dataGateMessage instanceof BatchChangeContinue) { - BatchChangeContinue batchChangeContinue = (BatchChangeContinue) dataGateMessage; - handleBatchChangeContinue(session, batchChangeContinue); - } else if (dataGateMessage instanceof BatchChangeEnd) { - BatchChangeEnd batchChangeEnd = (BatchChangeEnd) dataGateMessage; - handleBatchChangeEnd(session, batchChangeEnd); - } else if (dataGateMessage instanceof DataGateFeed) { - DataGateFeed dataGateFeed = (DataGateFeed) dataGateMessage; - handleDataGateFeed(session, dataGateFeed); - } else if (dataGateMessage instanceof Disconnect) { - Disconnect disconnect = (Disconnect) dataGateMessage; - handleDisconnect(session, disconnect); - } - } catch (Exception e) { - log.error("Error while handling message {}", message, e); - } - } - - @OnError - public void onError(Session session, Throwable ex) { - log.error("Error in DataGate server", ex); - - try { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setHeader(createHeader(MessageType.Error, null, null, - repository.getServerId(), "")); - errorMessage.setError(ex.getMessage()); - String message = objectMapper.writeValueAsString(errorMessage); - session.getBasicRemote().sendText(message); - } catch (Exception e) { - throw new ReplicationException("failed to send ErrorMessage", e, false); - } - } - - protected void handleConnect(Session session, Connect connect) throws IOException { - String replicaId = connect.getHeader().getOrigin(); - String userName = connect.getHeader().getUserName(); - String collection = userName + "@" + connect.getHeader().getCollection(); - - if (isValidAuth(userName, connect.getAuthToken())) { - session.getUserProperties().put("authorized", true); - session.getUserProperties().put("collection", collection); - session.getUserProperties().put("replica", replicaId); - - repository.getAuthorizedSessions().add(session); - - if (repository.getCollectionReplicaMap().containsKey(collection)) { - List replicas = repository.getCollectionReplicaMap().get(collection); - if (!replicas.contains(replicaId)) { - replicas.add(replicaId); - } - repository.getCollectionReplicaMap().put(collection, replicas); - } else { - List replicas = new ArrayList<>(); - replicas.add(replicaId); - repository.getCollectionReplicaMap().put(collection, replicas); - } - - if (repository.getUserReplicaMap().containsKey(userName)) { - List replicas = repository.getUserReplicaMap().get(userName); - if (!replicas.contains(replicaId)) { - replicas.add(replicaId); - } - repository.getUserReplicaMap().put(userName, replicas); - } else { - List replicas = new ArrayList<>(); - replicas.add(replicaId); - repository.getUserReplicaMap().put(userName, replicas); - } - - if (!repository.getReplicaStore().containsKey(collection)) { - LastWriteWinMap replica = createCrdt(collection); - repository.getReplicaStore().put(collection, replica); - } - - ConnectAck ack = new ConnectAck(); - ack.setHeader(createHeader(MessageType.ConnectAck, - connect.getHeader().getCollection(), userName, - repository.getServerId(), connect.getHeader().getId())); - ack.setTombstoneTtl(repository.getGcTtl()); - String message = objectMapper.writeValueAsString(ack); - session.getBasicRemote().sendText(message); - } else { - session.getUserProperties().put("authorized", false); - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setError("Unauthorized"); - errorMessage.setHeader(createHeader(MessageType.Error, - connect.getHeader().getCollection(), userName, - repository.getServerId(), connect.getHeader().getId())); - String message = objectMapper.writeValueAsString(errorMessage); - session.getBasicRemote().sendText(message); - } - } - - protected void handleDisconnect(Session session, Disconnect connect) { - String replicaId = connect.getHeader().getOrigin(); - String userName = connect.getHeader().getUserName(); - String collection = userName + "@" + connect.getHeader().getCollection(); - - repository.getCollectionReplicaMap().get(collection).remove(replicaId); - repository.getUserReplicaMap().get(userName).remove(replicaId); - repository.getAuthorizedSessions().remove(session); - } - - protected void handleDataGateFeed(Session channel, DataGateFeed feed) { - String userName = feed.getHeader().getUserName(); - String collection = userName + "@" + feed.getHeader().getCollection(); - String replicaId = feed.getHeader().getOrigin(); - - LastWriteWinMap replica = repository.getReplicaStore().get(collection); - replica.merge(feed.getFeed()); - - try { - Long syncTime = System.currentTimeMillis(); - String ackMessage = createAck(feed.getHeader().getCollection(), userName, - syncTime, feed.calculateReceipt(), feed.getHeader().getId()); - channel.getBasicRemote().sendText(ackMessage); - - // other peers will take this time as last sync times - feed.getHeader().setTimestamp(syncTime); - String message = objectMapper.writeValueAsString(feed); - broadcast(replicaId, collection, message); - } catch (Exception e) { - throw new ReplicationException("failed to broadcast DataGateFeed", e, false); - } - } - - private void broadcast(String origin, String collection, String message) { - repository.getAuthorizedSessions().stream() - .filter(s -> collection.equals(s.getUserProperties().get("collection"))) - .filter(s -> !origin.equals(s.getUserProperties().get("replica"))) - .forEach(s -> s.getAsyncRemote().sendText(message)); - } - - protected void handleBatchChangeEnd(Session session, BatchChangeEnd batchChangeEnd) throws IOException { - Long lastSync = batchChangeEnd.getLastSynced(); - Integer batchSize = batchChangeEnd.getBatchSize(); - Integer debounce = batchChangeEnd.getDebounce(); - String userName = batchChangeEnd.getHeader().getUserName(); - String collection = userName + "@" + batchChangeEnd.getHeader().getCollection(); - - BatchEndAck ack = new BatchEndAck(); - ack.setHeader(createHeader(MessageType.BatchEndAck, batchChangeEnd.getHeader().getCollection(), - userName, repository.getServerId(), batchChangeEnd.getHeader().getId())); - - String message = objectMapper.writeValueAsString(ack); - session.getBasicRemote().sendText(message); - - LastWriteWinMap replica = repository.getReplicaStore().get(collection); - sendChanges(batchChangeEnd.getHeader().getCollection(), userName, lastSync, - batchSize, debounce, replica, session, repository.getServerId()); - } - - protected void handleBatchChangeContinue(Session session, BatchChangeContinue batchChangeContinue) { - DataGateFeed feed = new DataGateFeed(); - - String userName = batchChangeContinue.getHeader().getUserName(); - String collection = userName + "@" + batchChangeContinue.getHeader().getCollection(); - String replicaId = batchChangeContinue.getHeader().getOrigin(); - LastWriteWinMap replica = repository.getReplicaStore().get(collection); - replica.merge(batchChangeContinue.getFeed()); - - feed.setHeader(createHeader(MessageType.DataGateFeed, batchChangeContinue.getHeader().getCollection(), - userName, replicaId, batchChangeContinue.getHeader().getId())); - feed.setFeed(batchChangeContinue.getFeed()); - - BatchAck ack = new BatchAck(); - ack.setReceipt(feed.calculateReceipt()); - ack.setHeader(createHeader(MessageType.BatchAck, batchChangeContinue.getHeader().getCollection(), - userName, repository.getServerId(), batchChangeContinue.getHeader().getId())); - - try { - String message = objectMapper.writeValueAsString(ack); - session.getBasicRemote().sendText(message); - - message = objectMapper.writeValueAsString(feed); - broadcast(replicaId, collection, message); - } catch (Exception e) { - throw new ReplicationException("failed to broadcast DataGateFeed", e, false); - } - } - - protected void handleBatchChangeStart(Session session, BatchChangeStart batchChangeStart) { - log.debug("BatchChangeStart message received " + batchChangeStart); - DataGateFeed feed = new DataGateFeed(); - - String userName = batchChangeStart.getHeader().getUserName(); - String collection = userName + "@" + batchChangeStart.getHeader().getCollection(); - String replicaId = batchChangeStart.getHeader().getOrigin(); - LastWriteWinMap replica = repository.getReplicaStore().get(collection); - replica.merge(batchChangeStart.getFeed()); - - feed.setHeader(createHeader(MessageType.DataGateFeed, batchChangeStart.getHeader().getCollection(), - userName, replicaId, batchChangeStart.getHeader().getId())); - feed.setFeed(batchChangeStart.getFeed()); - - BatchAck ack = new BatchAck(); - ack.setReceipt(feed.calculateReceipt()); - ack.setHeader(createHeader(MessageType.BatchAck, batchChangeStart.getHeader().getCollection(), - userName, repository.getServerId(), batchChangeStart.getHeader().getId())); - - try { - String message = objectMapper.writeValueAsString(ack); - session.getBasicRemote().sendText(message); - - message = objectMapper.writeValueAsString(feed); - broadcast(replicaId, collection, message); - } catch (Exception e) { - throw new ReplicationException("failed to broadcast DataGateFeed", e, false); - } - } - - private LastWriteWinMap createCrdt(String collection) { - NitriteCollection nc = repository.getDb().getCollection(collection); - NitriteMap nitriteMap = - repository.getDb().getConfig().getNitriteStore().openMap(collection + "-replica", - NitriteId.class, Long.class); - return new LastWriteWinMap(nc, nitriteMap); - } - - private void sendChanges(String collection, String userName, - Long lastSyncTime, Integer chunkSize, - Integer debounce, LastWriteWinMap crdt, - Session channel, String replicaId) { - try { - try { - String initMessage = createChangeStart(crdt, lastSyncTime, collection, userName, - chunkSize, debounce); - log.info("Sending BatchChangeStart message {} from server to {}", initMessage, replicaId); - channel.getBasicRemote().sendText(initMessage); - } catch (Exception e) { - log.error("Error while sending BatchChangeStart to " + replicaId, e); - } - - final Timer timer = new Timer(); - timer.scheduleAtFixedRate(new TimerTask() { - boolean hasMore = true; - int start = chunkSize; - - @Override - public void run() { - LastWriteWinState state = crdt.getChangesSince(lastSyncTime, start, chunkSize); - if (state.getChanges().size() == 0 && state.getTombstones().size() == 0) { - hasMore = false; - } - - if (hasMore) { - try { - String message = createChangeContinue(state, collection, userName, - chunkSize, debounce); - log.info("Sending BatchChangeContinue message {} from server to {}", message, replicaId); - channel.getBasicRemote().sendText(message); - } catch (Exception e) { - log.error("Error while sending BatchChangeContinue for " + replicaId, e); - } - - start = start + chunkSize; - } - - if (!hasMore) { - timer.cancel(); - } - } - }, 0, debounce); - - try { - String endMessage = createChangeEnd(collection, userName, - chunkSize, debounce); - log.info("Sending BatchChangeEnd message {} from server to {}", endMessage, replicaId); - channel.getBasicRemote().sendText(endMessage); - } catch (Exception e) { - log.error("Error while sending BatchChangeEnd for " + replicaId, e); - } - } catch (Exception e) { - throw new ReplicationException("failed to send local changes message for " + replicaId, e, false); - } - } - - private String createChangeStart(LastWriteWinMap crdt, Long lastSyncTime, - String collection, String userName, - Integer chunkSize, Integer debounce) { - try { - BatchChangeStart message = new BatchChangeStart(); - message.setHeader(createHeader(MessageType.BatchChangeStart, - collection, userName, repository.getServerId(), "")); - message.setBatchSize(chunkSize); - message.setDebounce(debounce); - - LastWriteWinState state = crdt.getChangesSince(lastSyncTime, 0, chunkSize); - message.setFeed(state); - return objectMapper.writeValueAsString(message); - } catch (JsonProcessingException e) { - throw new ReplicationException("failed to create BatchChangeStart message", e, false); - } - } - - private String createChangeContinue(LastWriteWinState state, - String collection, String userName, - Integer chunkSize, Integer debounce) { - try { - BatchChangeContinue message = new BatchChangeContinue(); - message.setHeader(createHeader(MessageType.BatchChangeContinue, - collection, userName, repository.getServerId(), "")); - message.setFeed(state); - message.setBatchSize(chunkSize); - message.setDebounce(debounce); - return objectMapper.writeValueAsString(message); - } catch (JsonProcessingException e) { - throw new ReplicationException("failed to create BatchChangeContinue message", e, false); - } - } - - private String createChangeEnd(String collection, String userName, - Integer chunkSize, Integer debounce) { - try { - BatchChangeEnd message = new BatchChangeEnd(); - message.setHeader(createHeader(MessageType.BatchChangeEnd, - collection, userName, repository.getServerId(), "")); - message.setLastSynced(System.currentTimeMillis()); - message.setBatchSize(chunkSize); - message.setDebounce(debounce); - return objectMapper.writeValueAsString(message); - } catch (JsonProcessingException e) { - throw new ReplicationException("failed to create BatchChangeEnd message", e, false); - } - } - - private String createAck(String collection, String userName, Long syncTime, Receipt receipt, String corrId) { - try { - DataGateFeedAck ack = new DataGateFeedAck(); - MessageHeader header = createHeader(MessageType.DataGateFeedAck, collection, - userName, repository.getServerId(), corrId); - header.setTimestamp(syncTime); - ack.setHeader(header); - ack.setReceipt(receipt); - return objectMapper.writeValueAsString(ack); - } catch (JsonProcessingException e) { - throw new ReplicationException("failed to create DataGateAck message", e, false); - } - } - - private MessageHeader createHeader(MessageType messageType, String collection, - String userName, String origin, String corrId) { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setId(UUID.randomUUID().toString()); - messageHeader.setCorrelationId(corrId); - messageHeader.setCollection(collection); - messageHeader.setMessageType(messageType); - messageHeader.setOrigin(origin); - messageHeader.setTimestamp(System.currentTimeMillis()); - messageHeader.setUserName(userName); - return messageHeader; - } - - private boolean isValidAuth(String userName, String authToken) { - return repository.getUserMap().get(userName).equals(authToken); - } -} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/SimpleDataGateServer.java b/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/SimpleDataGateServer.java deleted file mode 100644 index 3279f3aec..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/integration/server/SimpleDataGateServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.integration.server; - - -import org.glassfish.tyrus.server.Server; - -/** - * @author Anindya Chatterjee - */ -public class SimpleDataGateServer { - private final int port; - private Server server; - private final Repository repository; - - public SimpleDataGateServer(int port) { - this.port = port; - this.repository = Repository.getInstance(); - } - - public void start() throws Exception { - server = new Server("127.0.0.1", port, "", null, SimpleDataGateEndpoint.class); - server.start(); - Runtime.getRuntime().addShutdownHook(new Thread(() -> server.stop())); - } - - public void stop() { - server.stop(); - repository.reset(); - } -} diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/ConfigTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/ConfigTest.java deleted file mode 100644 index a138e75fa..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/ConfigTest.java +++ /dev/null @@ -1,203 +0,0 @@ -package org.dizitart.no2.sync; - -import com.fasterxml.jackson.databind.ObjectMapper; -import okhttp3.Request; -import org.junit.Test; - -import java.net.InetSocketAddress; -import java.net.Proxy; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.*; - -public class ConfigTest { - @Test - public void testCanEqual() { - assertFalse((new Config()).canEqual("other")); - } - - @Test - public void testEquals() { - Config config = new Config(); - config.setChunkSize(3); - assertFalse((new Config()).equals(config)); - } - - @Test - public void testEquals10() { - Config config = new Config(); - config.setChunkSize(3); - assertFalse(config.equals(new Config())); - } - - @Test - public void testEquals11() { - Config config = new Config(); - config.setUserName("janedoe"); - assertFalse((new Config()).equals(config)); - } - - @Test - public void testEquals12() { - Config config = new Config(); - config.setDebounce(0); - assertFalse(config.equals(new Config())); - } - - @Test - public void testEquals13() { - Config config = new Config(); - config.setDebounce(0); - assertFalse((new Config()).equals(config)); - } - - @Test - public void testEquals14() { - Config config = new Config(); - config.setObjectMapper(new ObjectMapper()); - assertFalse(config.equals(new Config())); - } - - @Test - public void testEquals15() { - Config config = new Config(); - config.setUserName("janedoe"); - assertFalse(config.equals(new Config())); - } - - @Test - public void testEquals2() { - Config config = new Config(); - config.setAuthToken("ABC123"); - assertFalse((new Config()).equals(config)); - } - - @Test - public void testEquals3() { - Config config = new Config(); - config.setObjectMapper(new ObjectMapper()); - assertFalse((new Config()).equals(config)); - } - - @Test - public void testEquals4() { - Config config = new Config(); - config.setAuthToken("ABC123"); - assertFalse(config.equals(new Config())); - } - - @Test - public void testEquals5() { - assertFalse((new Config()).equals("o")); - } - - @Test - public void testEquals6() { - Config config = new Config(); - assertTrue(config.equals(new Config())); - } - - @Test - public void testEquals7() { - Config config = new Config(); - config.setAcceptAllCertificates(true); - assertFalse(config.equals(new Config())); - } - - @Test - public void testEquals8() { - TimeSpan timeout = new TimeSpan(10L, TimeUnit.NANOSECONDS); - Config config = new Config(); - config.setTimeout(timeout); - assertFalse(config.equals(new Config())); - } - - @Test - public void testEquals9() { - Config config = new Config(); - config.setRequestBuilder(new Request.Builder()); - assertFalse(config.equals(new Config())); - } - - @Test - public void testSetAcceptAllCertificates() { - Config config = new Config(); - config.setAcceptAllCertificates(true); - assertTrue(config.isAcceptAllCertificates()); - } - - @Test - public void testSetAuthToken() { - Config config = new Config(); - config.setAuthToken("ABC123"); - assertEquals("ABC123", config.getAuthToken()); - } - - @Test - public void testSetChunkSize() { - Config config = new Config(); - config.setChunkSize(3); - assertEquals(3, config.getChunkSize().intValue()); - } - - @Test - public void testSetCollection() { - Config config = new Config(); - config.setCollection(null); - assertNull(config.getCollection()); - } - - @Test - public void testSetDebounce() { - Config config = new Config(); - config.setDebounce(1); - assertEquals(1, config.getDebounce().intValue()); - } - - @Test - public void testSetProxy() { - Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(1)); - Config config = new Config(); - config.setProxy(proxy); - assertEquals( - "Config(collection=null, chunkSize=null, userName=null, debounce=null, objectMapper=null, timeout=null," - + " requestBuilder=null, proxy=HTTP @ 0.0.0.0/0.0.0.0:1, authToken=null, acceptAllCertificates=false," - + " networkConnectivityChecker=null)", - config.toString()); - } - - @Test - public void testSetRequestBuilder() { - Config config = new Config(); - Request.Builder builder = new Request.Builder(); - config.setRequestBuilder(builder); - assertSame(builder, config.getRequestBuilder()); - } - - @Test - public void testSetTimeout() { - TimeSpan timeout = new TimeSpan(10L, TimeUnit.NANOSECONDS); - Config config = new Config(); - config.setTimeout(timeout); - assertEquals("Config(collection=null, chunkSize=null, userName=null, debounce=null, objectMapper=null, timeout" - + "=TimeSpan(time=10, timeUnit=NANOSECONDS), requestBuilder=null, proxy=null, authToken=null, acceptAll" - + "Certificates=false, networkConnectivityChecker=null)", config.toString()); - } - - @Test - public void testSetUserName() { - Config config = new Config(); - config.setUserName("janedoe"); - assertEquals("janedoe", config.getUserName()); - } - - @Test - public void testToString() { - assertEquals( - "Config(collection=null, chunkSize=null, userName=null, debounce=null, objectMapper=null, timeout=null," - + " requestBuilder=null, proxy=null, authToken=null, acceptAllCertificates=false, networkConnectivityChecker" - + "=null)", - (new Config()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/MessageFactoryTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/MessageFactoryTest.java index 1267dddbb..dca081245 100644 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/MessageFactoryTest.java +++ b/nitrite-replication/src/test/java/org/dizitart/no2/sync/MessageFactoryTest.java @@ -9,13 +9,14 @@ public class MessageFactoryTest { @Test public void testCreateHeader() { - MessageHeader actualCreateHeaderResult = (new MessageFactory()).createHeader(MessageType.Error, "collectionName", - "42", "42", "janedoe"); + MessageHeader actualCreateHeaderResult = (new MessageFactory()).createHeader(MessageType.Error, + "collectionName", "42", "42", "janedoe", "junit-test"); assertEquals("42", actualCreateHeaderResult.getOrigin()); assertEquals("collectionName", actualCreateHeaderResult.getCollection()); - assertEquals("42", actualCreateHeaderResult.getCorrelationId()); + assertEquals("42", actualCreateHeaderResult.getTransactionId()); assertEquals(MessageType.Error, actualCreateHeaderResult.getMessageType()); assertEquals("janedoe", actualCreateHeaderResult.getUserName()); + assertEquals("junit-test", actualCreateHeaderResult.getTenant()); } } diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/crdt/LastWriteWinStateTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/crdt/DeltaStatesTest.java similarity index 52% rename from nitrite-replication/src/test/java/org/dizitart/no2/sync/crdt/LastWriteWinStateTest.java rename to nitrite-replication/src/test/java/org/dizitart/no2/sync/crdt/DeltaStatesTest.java index ed1d986d9..586d4f110 100644 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/crdt/LastWriteWinStateTest.java +++ b/nitrite-replication/src/test/java/org/dizitart/no2/sync/crdt/DeltaStatesTest.java @@ -1,13 +1,13 @@ package org.dizitart.no2.sync.crdt; -import static org.junit.Assert.assertEquals; - import org.junit.Test; -public class LastWriteWinStateTest { +import static org.junit.Assert.assertEquals; + +public class DeltaStatesTest { @Test public void testConstructor() { - assertEquals("LastWriteWinState(changes=[], tombstones={})", (new LastWriteWinState()).toString()); + assertEquals("DeltaStates(changeSet=[], tombstoneMap={})", (new DeltaStates()).toString()); } } diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchAckTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchAckTest.java deleted file mode 100644 index e9060c196..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchAckTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class BatchAckTest { - @Test - public void testCanEqual() { - assertFalse((new BatchAck()).canEqual("other")); - } - - @Test - public void testEquals() { - BatchAck batchAck = new BatchAck(); - batchAck.setHeader(new MessageHeader()); - assertFalse((new BatchAck()).equals(batchAck)); - } - - @Test - public void testEquals2() { - BatchAck batchAck = new BatchAck(); - batchAck.setHeader(new MessageHeader()); - assertFalse(batchAck.equals(new BatchAck())); - } - - @Test - public void testEquals3() { - BatchAck batchAck = new BatchAck(); - batchAck.setReceipt(new Receipt()); - BatchAck batchAck1 = new BatchAck(); - batchAck1.setReceipt(new Receipt()); - assertTrue(batchAck.equals(batchAck1)); - } - - @Test - public void testEquals4() { - BatchAck batchAck = new BatchAck(); - batchAck.setReceipt(new Receipt()); - assertFalse((new BatchAck()).equals(batchAck)); - } - - @Test - public void testEquals5() { - BatchAck batchAck = new BatchAck(); - batchAck.setHeader(new MessageHeader()); - BatchAck batchAck1 = new BatchAck(); - batchAck1.setHeader(new MessageHeader()); - assertTrue(batchAck.equals(batchAck1)); - } - - @Test - public void testEquals6() { - BatchAck batchAck = new BatchAck(); - assertTrue(batchAck.equals(new BatchAck())); - } - - @Test - public void testEquals7() { - BatchAck batchAck = new BatchAck(); - batchAck.setReceipt(new Receipt()); - assertFalse(batchAck.equals(new BatchAck())); - } - - @Test - public void testEquals8() { - assertFalse((new BatchAck()).equals("o")); - } - - @Test - public void testSetHeader() { - BatchAck batchAck = new BatchAck(); - batchAck.setHeader(new MessageHeader()); - assertEquals( - "BatchAck(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null, timestamp=null," - + " messageType=null, origin=null), receipt=null)", - batchAck.toString()); - } - - @Test - public void testSetReceipt() { - BatchAck batchAck = new BatchAck(); - batchAck.setReceipt(new Receipt()); - assertEquals("BatchAck(header=null, receipt=Receipt(added=[], removed=[]))", batchAck.toString()); - } - - @Test - public void testToString() { - assertEquals("BatchAck(header=null, receipt=null)", (new BatchAck()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeContinueTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeContinueTest.java deleted file mode 100644 index 1d32935ab..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeContinueTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.junit.Test; - -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; - -public class BatchChangeContinueTest { - @Test - public void testCanEqual() { - assertFalse((new BatchChangeContinue()).canEqual("other")); - } - - @Test - public void testEquals() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setHeader(new MessageHeader()); - assertFalse(batchChangeContinue.equals(new BatchChangeContinue())); - } - - @Test - public void testEquals10() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setDebounce(0); - assertFalse((new BatchChangeContinue()).equals(batchChangeContinue)); - } - - @Test - public void testEquals2() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setDebounce(0); - assertFalse(batchChangeContinue.equals(new BatchChangeContinue())); - } - - @Test - public void testEquals3() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setHeader(new MessageHeader()); - assertFalse((new BatchChangeContinue()).equals(batchChangeContinue)); - } - - @Test - public void testEquals4() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setBatchSize(3); - assertFalse(batchChangeContinue.equals(new BatchChangeContinue())); - } - - @Test - public void testEquals5() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - assertTrue(batchChangeContinue.equals(new BatchChangeContinue())); - } - - @Test - public void testEquals6() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setFeed(new LastWriteWinState()); - assertFalse(batchChangeContinue.equals(new BatchChangeContinue())); - } - - @Test - public void testEquals7() { - assertFalse((new BatchChangeContinue()).equals("o")); - } - - @Test - public void testEquals8() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setFeed(new LastWriteWinState()); - assertFalse((new BatchChangeContinue()).equals(batchChangeContinue)); - } - - @Test - public void testEquals9() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setBatchSize(3); - assertFalse((new BatchChangeContinue()).equals(batchChangeContinue)); - } - - @Test - public void testSetBatchSize() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setBatchSize(3); - assertEquals(3, batchChangeContinue.getBatchSize().intValue()); - } - - @Test - public void testSetDebounce() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setDebounce(1); - assertEquals(1, batchChangeContinue.getDebounce().intValue()); - } - - @Test - public void testSetFeed() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setFeed(new LastWriteWinState()); - assertEquals("BatchChangeContinue(header=null, feed=LastWriteWinState(changes=[], tombstones={}), batchSize=null," - + " debounce=null)", batchChangeContinue.toString()); - } - - @Test - public void testSetHeader() { - BatchChangeContinue batchChangeContinue = new BatchChangeContinue(); - batchChangeContinue.setHeader(new MessageHeader()); - assertEquals( - "BatchChangeContinue(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null), feed=null, batchSize=null, debounce=null)", - batchChangeContinue.toString()); - } - - @Test - public void testToString() { - assertEquals("BatchChangeContinue(header=null, feed=null, batchSize=null, debounce=null)", - (new BatchChangeContinue()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeEndTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeEndTest.java deleted file mode 100644 index f240efc3d..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeEndTest.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class BatchChangeEndTest { - @Test - public void testCanEqual() { - assertFalse((new BatchChangeEnd()).canEqual("other")); - } - - @Test - public void testEquals() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setLastSynced(0L); - assertFalse((new BatchChangeEnd()).equals(batchChangeEnd)); - } - - @Test - public void testEquals10() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setLastSynced(0L); - assertFalse(batchChangeEnd.equals(new BatchChangeEnd())); - } - - @Test - public void testEquals2() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setDebounce(0); - assertFalse((new BatchChangeEnd()).equals(batchChangeEnd)); - } - - @Test - public void testEquals3() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - assertTrue(batchChangeEnd.equals(new BatchChangeEnd())); - } - - @Test - public void testEquals4() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setHeader(new MessageHeader()); - assertFalse(batchChangeEnd.equals(new BatchChangeEnd())); - } - - @Test - public void testEquals5() { - assertFalse((new BatchChangeEnd()).equals("o")); - } - - @Test - public void testEquals6() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setBatchSize(3); - assertFalse(batchChangeEnd.equals(new BatchChangeEnd())); - } - - @Test - public void testEquals7() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setBatchSize(3); - assertFalse((new BatchChangeEnd()).equals(batchChangeEnd)); - } - - @Test - public void testEquals8() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setHeader(new MessageHeader()); - assertFalse((new BatchChangeEnd()).equals(batchChangeEnd)); - } - - @Test - public void testEquals9() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setDebounce(0); - assertFalse(batchChangeEnd.equals(new BatchChangeEnd())); - } - - @Test - public void testSetBatchSize() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setBatchSize(3); - assertEquals(3, batchChangeEnd.getBatchSize().intValue()); - } - - @Test - public void testSetDebounce() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setDebounce(1); - assertEquals(1, batchChangeEnd.getDebounce().intValue()); - } - - @Test - public void testSetHeader() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setHeader(new MessageHeader()); - assertEquals( - "BatchChangeEnd(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null), lastSynced=null, batchSize=null, debounce=null)", - batchChangeEnd.toString()); - } - - @Test - public void testSetLastSynced() { - BatchChangeEnd batchChangeEnd = new BatchChangeEnd(); - batchChangeEnd.setLastSynced(1L); - assertEquals(1L, batchChangeEnd.getLastSynced().longValue()); - } - - @Test - public void testToString() { - assertEquals("BatchChangeEnd(header=null, lastSynced=null, batchSize=null, debounce=null)", - (new BatchChangeEnd()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeStartTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeStartTest.java deleted file mode 100644 index f47dfa797..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchChangeStartTest.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.junit.Test; - -import static org.junit.Assert.*; - -public class BatchChangeStartTest { - @Test - public void testCanEqual() { - assertFalse((new BatchChangeStart()).canEqual("other")); - } - - @Test - public void testEquals() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setDebounce(0); - assertFalse((new BatchChangeStart()).equals(batchChangeStart)); - } - - @Test - public void testEquals10() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setFeed(new LastWriteWinState()); - assertFalse(batchChangeStart.equals(new BatchChangeStart())); - } - - @Test - public void testEquals2() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setHeader(new MessageHeader()); - assertFalse(batchChangeStart.equals(new BatchChangeStart())); - } - - @Test - public void testEquals3() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setFeed(new LastWriteWinState()); - assertFalse((new BatchChangeStart()).equals(batchChangeStart)); - } - - @Test - public void testEquals4() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setBatchSize(3); - assertFalse(batchChangeStart.equals(new BatchChangeStart())); - } - - @Test - public void testEquals5() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setHeader(new MessageHeader()); - assertFalse((new BatchChangeStart()).equals(batchChangeStart)); - } - - @Test - public void testEquals6() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setBatchSize(3); - assertFalse((new BatchChangeStart()).equals(batchChangeStart)); - } - - @Test - public void testEquals7() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setDebounce(0); - assertFalse(batchChangeStart.equals(new BatchChangeStart())); - } - - @Test - public void testEquals8() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - assertTrue(batchChangeStart.equals(new BatchChangeStart())); - } - - @Test - public void testEquals9() { - assertFalse((new BatchChangeStart()).equals("o")); - } - - @Test - public void testSetBatchSize() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setBatchSize(3); - assertEquals(3, batchChangeStart.getBatchSize().intValue()); - } - - @Test - public void testSetDebounce() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setDebounce(1); - assertEquals(1, batchChangeStart.getDebounce().intValue()); - } - - @Test - public void testSetFeed() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setFeed(new LastWriteWinState()); - assertEquals("BatchChangeStart(header=null, batchSize=null, debounce=null, feed=LastWriteWinState(changes=[]," - + " tombstones={}))", batchChangeStart.toString()); - } - - @Test - public void testSetHeader() { - BatchChangeStart batchChangeStart = new BatchChangeStart(); - batchChangeStart.setHeader(new MessageHeader()); - assertEquals( - "BatchChangeStart(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null), batchSize=null, debounce=null, feed=null)", - batchChangeStart.toString()); - } - - @Test - public void testToString() { - assertEquals("BatchChangeStart(header=null, batchSize=null, debounce=null, feed=null)", - (new BatchChangeStart()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchEndAckTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchEndAckTest.java deleted file mode 100644 index 14c279aa9..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/BatchEndAckTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class BatchEndAckTest { - @Test - public void testCanEqual() { - assertFalse((new BatchEndAck()).canEqual("other")); - } - - @Test - public void testEquals() { - BatchEndAck batchEndAck = new BatchEndAck(); - batchEndAck.setHeader(new MessageHeader()); - assertFalse(batchEndAck.equals(new BatchEndAck())); - } - - @Test - public void testEquals2() { - BatchEndAck batchEndAck = new BatchEndAck(); - batchEndAck.setHeader(new MessageHeader()); - assertFalse((new BatchEndAck()).equals(batchEndAck)); - } - - @Test - public void testEquals3() { - BatchEndAck batchEndAck = new BatchEndAck(); - assertTrue(batchEndAck.equals(new BatchEndAck())); - } - - @Test - public void testEquals4() { - BatchEndAck batchEndAck = new BatchEndAck(); - batchEndAck.setHeader(new MessageHeader()); - BatchEndAck batchEndAck1 = new BatchEndAck(); - batchEndAck1.setHeader(new MessageHeader()); - assertTrue(batchEndAck.equals(batchEndAck1)); - } - - @Test - public void testEquals5() { - assertFalse((new BatchEndAck()).equals("o")); - } - - @Test - public void testSetHeader() { - BatchEndAck batchEndAck = new BatchEndAck(); - batchEndAck.setHeader(new MessageHeader()); - assertEquals("BatchEndAck(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null))", batchEndAck.toString()); - } - - @Test - public void testToString() { - assertEquals("BatchEndAck(header=null)", (new BatchEndAck()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ConnectAckTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ConnectAckTest.java deleted file mode 100644 index 94873cf1c..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ConnectAckTest.java +++ /dev/null @@ -1,94 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class ConnectAckTest { - @Test - public void testCanEqual() { - assertFalse((new ConnectAck()).canEqual("other")); - } - - @Test - public void testEquals() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setTombstoneTtl(0L); - assertFalse(connectAck.equals(new ConnectAck())); - } - - @Test - public void testEquals2() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setTombstoneTtl(0L); - assertFalse((new ConnectAck()).equals(connectAck)); - } - - @Test - public void testEquals3() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setHeader(new MessageHeader()); - ConnectAck connectAck1 = new ConnectAck(); - connectAck1.setHeader(new MessageHeader()); - assertTrue(connectAck.equals(connectAck1)); - } - - @Test - public void testEquals4() { - ConnectAck connectAck = new ConnectAck(); - assertTrue(connectAck.equals(new ConnectAck())); - } - - @Test - public void testEquals5() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setTombstoneTtl(1L); - ConnectAck connectAck1 = new ConnectAck(); - connectAck1.setTombstoneTtl(1L); - assertTrue(connectAck.equals(connectAck1)); - } - - @Test - public void testEquals6() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setHeader(new MessageHeader()); - assertFalse((new ConnectAck()).equals(connectAck)); - } - - @Test - public void testEquals7() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setHeader(new MessageHeader()); - assertFalse(connectAck.equals(new ConnectAck())); - } - - @Test - public void testEquals8() { - assertFalse((new ConnectAck()).equals("o")); - } - - @Test - public void testSetHeader() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setHeader(new MessageHeader()); - assertEquals( - "ConnectAck(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null, timestamp=null," - + " messageType=null, origin=null), tombstoneTtl=null)", - connectAck.toString()); - } - - @Test - public void testSetTombstoneTtl() { - ConnectAck connectAck = new ConnectAck(); - connectAck.setTombstoneTtl(1L); - assertEquals(1L, connectAck.getTombstoneTtl().longValue()); - } - - @Test - public void testToString() { - assertEquals("ConnectAck(header=null, tombstoneTtl=null)", (new ConnectAck()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ConnectTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ConnectTest.java deleted file mode 100644 index d885e16c2..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ConnectTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class ConnectTest { - @Test - public void testCanEqual() { - assertFalse((new Connect()).canEqual("other")); - } - - @Test - public void testEquals() { - Connect connect = new Connect(); - connect.setAuthToken("ABC123"); - assertFalse(connect.equals(new Connect())); - } - - @Test - public void testEquals2() { - Connect connect = new Connect(); - assertTrue(connect.equals(new Connect())); - } - - @Test - public void testEquals3() { - Connect connect = new Connect(); - connect.setAuthToken("ABC123"); - assertFalse((new Connect()).equals(connect)); - } - - @Test - public void testEquals4() { - Connect connect = new Connect(); - connect.setHeader(new MessageHeader()); - assertFalse(connect.equals(new Connect())); - } - - @Test - public void testEquals5() { - Connect connect = new Connect(); - connect.setHeader(new MessageHeader()); - assertFalse((new Connect()).equals(connect)); - } - - @Test - public void testEquals6() { - Connect connect = new Connect(); - connect.setHeader(new MessageHeader()); - Connect connect1 = new Connect(); - connect1.setHeader(new MessageHeader()); - assertTrue(connect.equals(connect1)); - } - - @Test - public void testEquals7() { - Connect connect = new Connect(); - connect.setAuthToken("ABC123"); - Connect connect1 = new Connect(); - connect1.setAuthToken("ABC123"); - connect1.setHeader(null); - assertTrue(connect.equals(connect1)); - } - - @Test - public void testEquals8() { - assertFalse((new Connect()).equals("o")); - } - - @Test - public void testSetAuthToken() { - Connect connect = new Connect(); - connect.setAuthToken("ABC123"); - assertEquals("ABC123", connect.getAuthToken()); - } - - @Test - public void testSetHeader() { - Connect connect = new Connect(); - connect.setHeader(new MessageHeader()); - assertEquals( - "Connect(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null, timestamp=null," - + " messageType=null, origin=null), authToken=null)", - connect.toString()); - } - - @Test - public void testToString() { - assertEquals("Connect(header=null, authToken=null)", (new Connect()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DataGateFeedAckTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DataGateFeedAckTest.java deleted file mode 100644 index 9d5e8a770..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DataGateFeedAckTest.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class DataGateFeedAckTest { - @Test - public void testCanEqual() { - assertFalse((new DataGateFeedAck()).canEqual("other")); - } - - @Test - public void testEquals() { - assertFalse((new DataGateFeedAck()).equals("o")); - } - - @Test - public void testEquals2() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - assertTrue(dataGateFeedAck.equals(new DataGateFeedAck())); - } - - @Test - public void testEquals3() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setHeader(new MessageHeader()); - assertFalse((new DataGateFeedAck()).equals(dataGateFeedAck)); - } - - @Test - public void testEquals4() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setReceipt(new Receipt()); - assertFalse((new DataGateFeedAck()).equals(dataGateFeedAck)); - } - - @Test - public void testEquals5() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setHeader(new MessageHeader()); - DataGateFeedAck dataGateFeedAck1 = new DataGateFeedAck(); - dataGateFeedAck1.setHeader(new MessageHeader()); - assertTrue(dataGateFeedAck.equals(dataGateFeedAck1)); - } - - @Test - public void testEquals6() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setHeader(new MessageHeader()); - assertFalse(dataGateFeedAck.equals(new DataGateFeedAck())); - } - - @Test - public void testEquals7() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setReceipt(new Receipt()); - DataGateFeedAck dataGateFeedAck1 = new DataGateFeedAck(); - dataGateFeedAck1.setReceipt(new Receipt()); - assertTrue(dataGateFeedAck.equals(dataGateFeedAck1)); - } - - @Test - public void testEquals8() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setReceipt(new Receipt()); - assertFalse(dataGateFeedAck.equals(new DataGateFeedAck())); - } - - @Test - public void testSetHeader() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setHeader(new MessageHeader()); - assertEquals("DataGateFeedAck(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null), receipt=null)", dataGateFeedAck.toString()); - } - - @Test - public void testSetReceipt() { - DataGateFeedAck dataGateFeedAck = new DataGateFeedAck(); - dataGateFeedAck.setReceipt(new Receipt()); - assertEquals("DataGateFeedAck(header=null, receipt=Receipt(added=[], removed=[]))", dataGateFeedAck.toString()); - } - - @Test - public void testToString() { - assertEquals("DataGateFeedAck(header=null, receipt=null)", (new DataGateFeedAck()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DataGateFeedTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DataGateFeedTest.java deleted file mode 100644 index 4f096a567..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DataGateFeedTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.dizitart.no2.sync.crdt.LastWriteWinState; -import org.junit.Test; - -import static org.junit.Assert.*; - -public class DataGateFeedTest { - @Test - public void testCanEqual() { - assertFalse((new DataGateFeed()).canEqual("other")); - } - - @Test - public void testEquals() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setHeader(new MessageHeader()); - DataGateFeed dataGateFeed1 = new DataGateFeed(); - dataGateFeed1.setHeader(new MessageHeader()); - assertTrue(dataGateFeed.equals(dataGateFeed1)); - } - - @Test - public void testEquals2() { - DataGateFeed dataGateFeed = new DataGateFeed(); - assertTrue(dataGateFeed.equals(new DataGateFeed())); - } - - @Test - public void testEquals3() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setFeed(new LastWriteWinState()); - DataGateFeed dataGateFeed1 = new DataGateFeed(); - dataGateFeed1.setFeed(new LastWriteWinState()); - assertTrue(dataGateFeed.equals(dataGateFeed1)); - } - - @Test - public void testEquals4() { - assertFalse((new DataGateFeed()).equals("o")); - } - - @Test - public void testEquals5() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setFeed(new LastWriteWinState()); - assertFalse((new DataGateFeed()).equals(dataGateFeed)); - } - - @Test - public void testEquals6() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setFeed(new LastWriteWinState()); - assertFalse(dataGateFeed.equals(new DataGateFeed())); - } - - @Test - public void testEquals7() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setHeader(new MessageHeader()); - assertFalse((new DataGateFeed()).equals(dataGateFeed)); - } - - @Test - public void testEquals8() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setHeader(new MessageHeader()); - assertFalse(dataGateFeed.equals(new DataGateFeed())); - } - - @Test - public void testSetFeed() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setFeed(new LastWriteWinState()); - assertEquals("DataGateFeed(header=null, feed=LastWriteWinState(changes=[], tombstones={}))", - dataGateFeed.toString()); - } - - @Test - public void testSetHeader() { - DataGateFeed dataGateFeed = new DataGateFeed(); - dataGateFeed.setHeader(new MessageHeader()); - assertEquals("DataGateFeed(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null), feed=null)", dataGateFeed.toString()); - } - - @Test - public void testToString() { - assertEquals("DataGateFeed(header=null, feed=null)", (new DataGateFeed()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DisconnectAckTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DisconnectAckTest.java deleted file mode 100644 index ef41de26d..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DisconnectAckTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class DisconnectAckTest { - @Test - public void testCanEqual() { - assertFalse((new DisconnectAck()).canEqual("other")); - } - - @Test - public void testEquals() { - DisconnectAck disconnectAck = new DisconnectAck(); - disconnectAck.setHeader(new MessageHeader()); - DisconnectAck disconnectAck1 = new DisconnectAck(); - disconnectAck1.setHeader(new MessageHeader()); - assertTrue(disconnectAck.equals(disconnectAck1)); - } - - @Test - public void testEquals2() { - assertFalse((new DisconnectAck()).equals("o")); - } - - @Test - public void testEquals3() { - DisconnectAck disconnectAck = new DisconnectAck(); - disconnectAck.setHeader(new MessageHeader()); - assertFalse((new DisconnectAck()).equals(disconnectAck)); - } - - @Test - public void testEquals4() { - DisconnectAck disconnectAck = new DisconnectAck(); - disconnectAck.setHeader(new MessageHeader()); - assertFalse(disconnectAck.equals(new DisconnectAck())); - } - - @Test - public void testEquals5() { - DisconnectAck disconnectAck = new DisconnectAck(); - assertTrue(disconnectAck.equals(new DisconnectAck())); - } - - @Test - public void testSetHeader() { - DisconnectAck disconnectAck = new DisconnectAck(); - disconnectAck.setHeader(new MessageHeader()); - assertEquals("DisconnectAck(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null))", disconnectAck.toString()); - } - - @Test - public void testToString() { - assertEquals("DisconnectAck(header=null)", (new DisconnectAck()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DisconnectTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DisconnectTest.java deleted file mode 100644 index 515313eca..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/DisconnectTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class DisconnectTest { - @Test - public void testCanEqual() { - assertFalse((new Disconnect()).canEqual("other")); - } - - @Test - public void testEquals() { - Disconnect disconnect = new Disconnect(); - disconnect.setHeader(new MessageHeader()); - assertFalse(disconnect.equals(new Disconnect())); - } - - @Test - public void testEquals2() { - Disconnect disconnect = new Disconnect(); - disconnect.setHeader(new MessageHeader()); - Disconnect disconnect1 = new Disconnect(); - disconnect1.setHeader(new MessageHeader()); - assertTrue(disconnect.equals(disconnect1)); - } - - @Test - public void testEquals3() { - Disconnect disconnect = new Disconnect(); - assertTrue(disconnect.equals(new Disconnect())); - } - - @Test - public void testEquals4() { - Disconnect disconnect = new Disconnect(); - disconnect.setHeader(new MessageHeader()); - assertFalse((new Disconnect()).equals(disconnect)); - } - - @Test - public void testEquals5() { - assertFalse((new Disconnect()).equals("o")); - } - - @Test - public void testSetHeader() { - Disconnect disconnect = new Disconnect(); - disconnect.setHeader(new MessageHeader()); - assertEquals( - "Disconnect(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null, timestamp=null," - + " messageType=null, origin=null))", - disconnect.toString()); - } - - @Test - public void testToString() { - assertEquals("Disconnect(header=null)", (new Disconnect()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ErrorMessageTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ErrorMessageTest.java deleted file mode 100644 index e2159667a..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ErrorMessageTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class ErrorMessageTest { - @Test - public void testCanEqual() { - assertFalse((new ErrorMessage()).canEqual("other")); - } - - @Test - public void testEquals() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setHeader(new MessageHeader()); - assertFalse((new ErrorMessage()).equals(errorMessage)); - } - - @Test - public void testEquals2() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setHeader(new MessageHeader()); - ErrorMessage errorMessage1 = new ErrorMessage(); - errorMessage1.setHeader(new MessageHeader()); - assertTrue(errorMessage.equals(errorMessage1)); - } - - @Test - public void testEquals3() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setError("An error occurred"); - assertFalse(errorMessage.equals(new ErrorMessage())); - } - - @Test - public void testEquals4() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setError("An error occurred"); - ErrorMessage errorMessage1 = new ErrorMessage(); - errorMessage1.setError("An error occurred"); - errorMessage1.setHeader(null); - assertTrue(errorMessage.equals(errorMessage1)); - } - - @Test - public void testEquals5() { - ErrorMessage errorMessage = new ErrorMessage(); - assertTrue(errorMessage.equals(new ErrorMessage())); - } - - @Test - public void testEquals6() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setError("An error occurred"); - assertFalse((new ErrorMessage()).equals(errorMessage)); - } - - @Test - public void testEquals7() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setHeader(new MessageHeader()); - assertFalse(errorMessage.equals(new ErrorMessage())); - } - - @Test - public void testEquals8() { - assertFalse((new ErrorMessage()).equals("o")); - } - - @Test - public void testSetError() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setError("An error occurred"); - assertEquals("An error occurred", errorMessage.getError()); - } - - @Test - public void testSetHeader() { - ErrorMessage errorMessage = new ErrorMessage(); - errorMessage.setHeader(new MessageHeader()); - assertEquals("ErrorMessage(header=MessageHeader(id=null, correlationId=null, collection=null, userName=null," - + " timestamp=null, messageType=null, origin=null), error=null)", errorMessage.toString()); - } - - @Test - public void testToString() { - assertEquals("ErrorMessage(header=null, error=null)", (new ErrorMessage()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/MessageHeaderTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/MessageHeaderTest.java deleted file mode 100644 index c25930cd3..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/MessageHeaderTest.java +++ /dev/null @@ -1,180 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; - -public class MessageHeaderTest { - @Test - public void testCanEqual() { - assertFalse((new MessageHeader()).canEqual("other")); - } - - @Test - public void testEquals() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setId("42"); - assertFalse(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals10() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setUserName("janedoe"); - assertFalse(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals11() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setCorrelationId("42"); - assertFalse((new MessageHeader()).equals(messageHeader)); - } - - @Test - public void testEquals12() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setCorrelationId("42"); - assertFalse(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals13() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setCollection("collection"); - assertFalse(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals14() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setOrigin("origin"); - assertFalse((new MessageHeader()).equals(messageHeader)); - } - - @Test - public void testEquals15() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setCollection("collection"); - assertFalse((new MessageHeader()).equals(messageHeader)); - } - - @Test - public void testEquals16() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setId("42"); - assertFalse((new MessageHeader()).equals(messageHeader)); - } - - @Test - public void testEquals2() { - MessageHeader messageHeader = new MessageHeader(); - assertTrue(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals3() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setUserName("janedoe"); - assertFalse((new MessageHeader()).equals(messageHeader)); - } - - @Test - public void testEquals4() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setMessageType(MessageType.Error); - assertFalse(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals5() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setMessageType(MessageType.Error); - assertFalse((new MessageHeader()).equals(messageHeader)); - } - - @Test - public void testEquals6() { - assertFalse((new MessageHeader()).equals("o")); - } - - @Test - public void testEquals7() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setOrigin("origin"); - assertFalse(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals8() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setTimestamp(10L); - assertFalse(messageHeader.equals(new MessageHeader())); - } - - @Test - public void testEquals9() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setTimestamp(10L); - assertFalse((new MessageHeader()).equals(messageHeader)); - } - - @Test - public void testSetCollection() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setCollection("collection"); - assertEquals("collection", messageHeader.getCollection()); - } - - @Test - public void testSetCorrelationId() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setCorrelationId("42"); - assertEquals("42", messageHeader.getCorrelationId()); - } - - @Test - public void testSetId() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setId("42"); - assertEquals("42", messageHeader.getId()); - } - - @Test - public void testSetMessageType() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setMessageType(MessageType.Error); - assertEquals(MessageType.Error, messageHeader.getMessageType()); - } - - @Test - public void testSetOrigin() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setOrigin("origin"); - assertEquals("origin", messageHeader.getOrigin()); - } - - @Test - public void testSetTimestamp() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setTimestamp(10L); - assertEquals(10L, messageHeader.getTimestamp().longValue()); - } - - @Test - public void testSetUserName() { - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setUserName("janedoe"); - assertEquals("janedoe", messageHeader.getUserName()); - } - - @Test - public void testToString() { - assertEquals( - "MessageHeader(id=null, correlationId=null, collection=null, userName=null, timestamp=null, messageType=null," - + " origin=null)", - (new MessageHeader()).toString()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/MessageTypeTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/MessageTypeTest.java deleted file mode 100644 index 25a5d5173..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/MessageTypeTest.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.dizitart.no2.sync.message; - -import static org.junit.Assert.assertEquals; - -import org.junit.Test; - -public class MessageTypeTest { - @Test - public void testCode() { - assertEquals("no2.sync.error", MessageType.Error.code()); - } -} - diff --git a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ReceiptTest.java b/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ReceiptTest.java deleted file mode 100644 index 832a5052a..000000000 --- a/nitrite-replication/src/test/java/org/dizitart/no2/sync/message/ReceiptTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.dizitart.no2.sync.message; - -import org.junit.Test; - -import java.util.HashSet; - -import static org.junit.Assert.*; - -public class ReceiptTest { - @Test - public void testCanEqual() { - assertFalse((new Receipt()).canEqual("other")); - } - - @Test - public void testConstructor() { - HashSet added = new HashSet(); - assertEquals("Receipt(added=[], removed=[])", (new Receipt(added, new HashSet())).toString()); - } - - @Test - public void testConstructor2() { - assertEquals("Receipt(added=[], removed=[])", (new Receipt()).toString()); - } - - @Test - public void testEquals() { - Receipt receipt = new Receipt(); - receipt.setRemoved(null); - assertFalse(receipt.equals(new Receipt())); - } - - @Test - public void testEquals2() { - assertFalse((new Receipt()).equals("o")); - } - - @Test - public void testEquals3() { - Receipt receipt = new Receipt(); - receipt.setAdded(null); - assertFalse(receipt.equals(new Receipt())); - } - - @Test - public void testEquals4() { - Receipt receipt = new Receipt(); - receipt.setRemoved(null); - assertFalse((new Receipt()).equals(receipt)); - } - - @Test - public void testEquals5() { - Receipt receipt = new Receipt(); - assertTrue(receipt.equals(new Receipt())); - } - - @Test - public void testEquals6() { - Receipt receipt = new Receipt(); - receipt.setAdded(null); - assertFalse((new Receipt()).equals(receipt)); - } - - @Test - public void testSetAdded() { - Receipt receipt = new Receipt(); - receipt.setAdded(new HashSet()); - assertEquals("Receipt(added=[], removed=[])", receipt.toString()); - } - - @Test - public void testSetRemoved() { - Receipt receipt = new Receipt(); - receipt.setRemoved(new HashSet()); - assertEquals("Receipt(added=[], removed=[])", receipt.toString()); - } - - @Test - public void testToString() { - assertEquals("Receipt(added=[], removed=[])", (new Receipt()).toString()); - } -} - diff --git a/nitrite-replication/src/test/resources/docker-compose.yml b/nitrite-replication/src/test/resources/docker-compose.yml new file mode 100644 index 000000000..c8f3ce76e --- /dev/null +++ b/nitrite-replication/src/test/resources/docker-compose.yml @@ -0,0 +1,38 @@ +version: '3.3' + +services: + mongo: + image: 'mongo' + container_name: 'mongo' + restart: always + ports: + - '27017:27017' + + mongo-express: + image: 'mongo-express' + container_name: 'mongo-express' + depends_on: + - 'mongo' + ports: + - '8081:8081' + restart: always + + mongo-seed: + depends_on: + - 'mongo' + build: ./mongo-seed + container_name: 'mongo-seed' + + datagate: + depends_on: + - 'mongo-seed' + image: 'nitrite/nitrite-datagate:latest' + container_name: 'nitrite-datagate' + ports: + - '46005:46005' + restart: always + environment: + MONGO_URL: mongodb://mongo:27017/datagate + GO_ENV: production + SERVER_PORT: 46005 + SERVER_HOST: 0.0.0.0 \ No newline at end of file diff --git a/nitrite-replication/src/test/resources/log4j2.xml b/nitrite-replication/src/test/resources/log4j2.xml deleted file mode 100644 index c30ab253c..000000000 --- a/nitrite-replication/src/test/resources/log4j2.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/nitrite-replication/src/test/resources/logback.xml b/nitrite-replication/src/test/resources/logback.xml new file mode 100644 index 000000000..f182a78c7 --- /dev/null +++ b/nitrite-replication/src/test/resources/logback.xml @@ -0,0 +1,30 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{15} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/nitrite-replication/src/test/resources/mongo-seed/Dockerfile b/nitrite-replication/src/test/resources/mongo-seed/Dockerfile new file mode 100644 index 000000000..0c9977d91 --- /dev/null +++ b/nitrite-replication/src/test/resources/mongo-seed/Dockerfile @@ -0,0 +1,6 @@ +FROM mongo + +COPY appConfig.json /appConfig.json + +CMD mongoimport --host mongo --db datagate --collection serverConfigs --drop --type json --file /serverConfig.json --jsonArray +CMD mongoimport --host mongo --db datagate --collection appConfigs --drop --type json --file /appConfig.json --jsonArray \ No newline at end of file diff --git a/nitrite-replication/src/test/resources/mongo-seed/appConfig.json b/nitrite-replication/src/test/resources/mongo-seed/appConfig.json new file mode 100644 index 000000000..e1904dc67 --- /dev/null +++ b/nitrite-replication/src/test/resources/mongo-seed/appConfig.json @@ -0,0 +1,62 @@ +[ + { + "app_name": "datagate", + "debug_message": false, + "tombstone_ttl": 0, + "auth_config": { + "init_user": "test", + "init_password": "test", + "internal_public_key": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQ0lqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FnOEFNSUlDQ2dLQ0FnRUE3TlJVL2FNWnF6Sm8vdnEyREFwMgpyenlWY1RPVHh0Mnh3VitnUUdDaGNWWlpsNVhDZk9FU1NEek5oZG1jTURnWUZ6MVpFd2loVUFDRDFKMzdXM0xRCkpWVzZVdHN1YTRMZG1COXgzci9teWV3OUMrRWZFNlBLY2pYSHU0NVBISTlRb1JDd1VzcDdGNE5PZElIWm12UjYKeFU0SkR0M09hcksvazVyaVpVVEd6WTJWbGxLUDdaL28wcXBGTmRDa2k3aDV6Vjd1TmlqMFFHbU5rN0hRa3FGMQpWSnIvU01oRStnTitUNmJiWVYrTUtqOWdtcmlRUDhyT0hYQUFjYkt0THRpSzlOM2FVWVhkM2Y0VnBWR0FaSlExCms1KzdvTWl5V2cxeTVGWWhFL1FyTTN3NlFtaWthUHAzci9LNUVXNUg2VUVCcEs2elRibU1JYUFJcVdMUnZ3ZjIKN3JFLy9mb0xhQ2ZEa1dFUGl5bGluKzFWOU4yM3JCMCtCWW1ta2VPcUt1OVpEdlUyV292VnRGeFdVQmU1TXNqSApPN2hGYzErUHEzcWU3bVZPNlVTdEdueUhJVEJjWVgrR3A3Qmo5amtrTEpoRS9OTHZ0akFQOFpQR1lCWXhZTHhNCjNndm9pN2RaaDFGU00vdUpGRjkzblRNNU0xWG1yR1MxaDZSbEFITE1ISUtSenZiYVRPRFZFdGtzUDNSQVUvVzEKZVZRYTVJUUd3a3FsZzdiNmYwcjNOZEcxbmhBWE1RZWo2QnZlOGIzZWozaHozQ2ZydnUwNnNFcjJqREF0UXZSNApZdGdUcDZ1Z1lxOVFqdENuRmR4RTNrdlVqNWxNMW1VODJhL09aYjRyZklISGFQTjR0TGJTZW1LZWJVZGliVzd0ClFWTktuMzNVLzdpbW5TQ2xQQ3ZHbGVVQ0F3RUFBUT09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo=", + "internal_private_key": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS1FJQkFBS0NBZ0VBN05SVS9hTVpxekpvL3ZxMkRBcDJyenlWY1RPVHh0Mnh3VitnUUdDaGNWWlpsNVhDCmZPRVNTRHpOaGRtY01EZ1lGejFaRXdpaFVBQ0QxSjM3VzNMUUpWVzZVdHN1YTRMZG1COXgzci9teWV3OUMrRWYKRTZQS2NqWEh1NDVQSEk5UW9SQ3dVc3A3RjROT2RJSFptdlI2eFU0SkR0M09hcksvazVyaVpVVEd6WTJWbGxLUAo3Wi9vMHFwRk5kQ2tpN2g1elY3dU5pajBRR21OazdIUWtxRjFWSnIvU01oRStnTitUNmJiWVYrTUtqOWdtcmlRClA4ck9IWEFBY2JLdEx0aUs5TjNhVVlYZDNmNFZwVkdBWkpRMWs1KzdvTWl5V2cxeTVGWWhFL1FyTTN3NlFtaWsKYVBwM3IvSzVFVzVINlVFQnBLNnpUYm1NSWFBSXFXTFJ2d2YyN3JFLy9mb0xhQ2ZEa1dFUGl5bGluKzFWOU4yMwpyQjArQlltbWtlT3FLdTlaRHZVMldvdlZ0RnhXVUJlNU1zakhPN2hGYzErUHEzcWU3bVZPNlVTdEdueUhJVEJjCllYK0dwN0JqOWpra0xKaEUvTkx2dGpBUDhaUEdZQll4WUx4TTNndm9pN2RaaDFGU00vdUpGRjkzblRNNU0xWG0KckdTMWg2UmxBSExNSElLUnp2YmFUT0RWRXRrc1AzUkFVL1cxZVZRYTVJUUd3a3FsZzdiNmYwcjNOZEcxbmhBWApNUWVqNkJ2ZThiM2VqM2h6M0NmcnZ1MDZzRXIyakRBdFF2UjRZdGdUcDZ1Z1lxOVFqdENuRmR4RTNrdlVqNWxNCjFtVTgyYS9PWmI0cmZJSEhhUE40dExiU2VtS2ViVWRpYlc3dFFWTktuMzNVLzdpbW5TQ2xQQ3ZHbGVVQ0F3RUEKQVFLQ0FnRUF4YnlYWmRxTE1ReDY3QUhxZyswc29TMkZYU29DUmJXT2wvVSt1T0cxaWdyZDdSbkdkRHY3NXNLVgpteDlSTUZWMWo5blNDSGxaTHBIdmdGT1RyZ3dUekVoaXRKRjZsWnVEWjJOQjBRa0xLaWNMNVdKR2IwQi9aSktRCnZJR2FmaThPMUJ3NkREWXhSaldGQ1BQdCsxb0xNN2Z2Q2pHYUNpNUtsSFJxZU1GTytBc3lEWHZMM2t4NHVZUWYKRzBxa1NHQnpta3lidWk3Qm1SSkllanVwK1BQRUlpcno0UklOdlcyelJjLzhOYlh3TXNvTjM4RWY5NU5lT3VmcwpCd0ozWkxpNmRLN1RmT08zbG9WeUQwRVlZV0g1eGRORmRuNTdvNEs5SGZibjBXQTV2ZGdJVUxCTUxiYUt2aVo3CjdSaldBK1FaK2lVL1lqTDgxSXBwRVB5SVFlYmxmVlRqQitQa3pKSWQ1R2dac0x6MlFka09xRnZWVE9pT0hNYlQKbDBIS3RmaFpod2pBRG1FeVdQaXIxOHNuVGs3SHoxbmh0eHV3ZGRlREZsZFZ2cU1xZlEyK25tTVBYRjdXbk9uWAppTGcyQlE0ZDFHS3c0WTBMR3FPdWl3ckFtOThlMEJLN2swdTN5WlVtRzRQRE91bzZFV2pyM0pCQ1p1YmViSVBvCkhhVnpBWllhTlZsbHN2dm9LOGp6eWN5aHo3cTBjK1lwRUJHLzNHTGFFbmdaSzVMUzRTZnhxU1FXc25aZjBmeE0KL0VBdWpEMUVjanZaTXJ6ejZ3SWxkd2I3Qkh6N3NBb1dXdmh1aUdZcEdWSzZSTUZhUDNmdXFvMkkwRVUrdEV5QQpOaW5mVDA3SkFGNnUwT1Zpc0dndVdiMHNGR2VKWis3UlVTc3RUbkYxQWh6dTgzdUM5N1VDZ2dFQkFQZzl4MXJqCjl5ZGhEaGFYWXQxNk9sQWU5NFR6eW41N2NzUlBlRlRSZjV4ZjF1MHVyQzJYZ2IvYnZ3MHI2SGc3bVBIRUV1NjMKU1ZxNUx2QzFEVHo5QzNJR1o3MkpyM1RWVTRsNlI2clVtNVBUYXExaE10M25WWlVLK2tUa0NWNE5wOVE3Vm0wMgpWT1IrSjZpM2JZOFU1bkZEemU1ZEk2QjRzWTFoeGQweW1wQW5Xc3dPSG5HMnFlclVEaHFHU2lJOFpzNmo0NTZJCmJZSkFkZ1BQRVZHNVY1V3owbnp2RG1YK3RQM05RRXF3NEE2MW9tQW9tVS8zODNQSVJ4MnJJSjRTT3JlUXNyd1QKdXBHTVJ0QWZ4bVV1aVJvRHI2eFNiczJ6SWRZcVgwcloxSnlBaXdqTUpNa2RBWWZmMGw1UUlwK1VVOTNYQ01nawpuSmFoc3JzRDdpcVluanNDZ2dFQkFQUTdQcVFTay9wY2ltdS9LSVhRSHhEN0N2Q2txY3I4UGpWTVo0NG5jMk5vCkZROEU2UzgxQmNFVEVDWmx4MVRyVkFVVmdFajhwTG9rTytHcUlzMGtHMHduQUNITVZLMS9nbE4vRCtSbm9zYXYKR1hmbnpnRGJzVVlNcDRCcDAwK1Uyc1ByWGFyOFU3ZWRZSnM2ZmhKcTAvbC85Y3BHcTByUHlncVZiWUd5KzBsaQpiR2NkMkxvTEhOdmdRS2dLZys1Zmg2WHFMU3lXWnJLcWQxVnhmSGVyZ01GUis4RXhodXNXRlpLTzFHZ1FPMzc2CjRsV0ZuUDBtMnJLMjlzSDJ4bWE4UVE0STFjRS95VEhBTDNOTUoyTCtTQjlHUy91ZGc5WGczdFI4dytZdW1EN3EKR09GeWdtTmZzR1NYYXpkRWxBTEFHRElENUEwd0UvRWdselNCanpxM3VsOENnZ0VBWTE2SGdMQ2tiTlVEQ0xRTQoxVTlxTEV4WkZKVnFSM3N2RTdva0Z2L05yMUVGL2VlaThKVW5VUitydUtBTTdLUWVzeGlqNDM3bkZEUHd3RllaCk9JS3FwRGhBS3JVRTBTWGJ6THB3R2NnRmh3VW9QTU1kMDRvWXpoS1k0QjdRU1IvNlFKQ0lKaXVMaS9PYitJT0UKamJQMkV2enJZREZVWTVZc3JNV29xTVRxN2kxeXdTQWR1N005RFUxWlgvREZtRExKaklvNlFXbW5QRzZGVHowQwpWODV6YXUrU29JUXBKVmJ5S0c2Uy85TVJ2Wkdqc0E1UVlKeUdqYUJzSjBvclFsdFZ1Y2xvWXJVYkI4dzVSSEtUCnZra0VoSzlaRVFmbVp0ei8vSFQxdEViQ1B1dU52RFhMdTkycWtUTmRTSGVYaEgyaG5MbkpRQ1Mzc2V5RVdTeFgKbUNHRHBRS0NBUUVBaTNyYVIzR2t1VExvaXFoZFNDNlh6MmJQMUtiMW9VdDFhNUw3QVNCZXNjTGJZL3gxLzlQVQpPWFBkb1ZBM0NyUnJBNHhIKzJidDNMQ2Mwa0FNS0FRYTR0N1RJSHBGVWVDa1dYTVRiR29UZUV5L3lzN0R3NUcwCktFRkoxL2lZQ2JjRlNTYStFOHlQTXluWjVrejlleDh2ZUNvd0FSbGk4aExCWEZJQ2ZEUHZkdldTMjBFY2FRTzMKczRyYTRoMC9RMytqUkluOHlwNEtnTGNCOS9ZY0Uyd0syRjB0M2lPZTNkdDY3bnhMcWpLN0I4WFlST2ROeFBYUApxSWo5VzhESGhoeTFPb0twTVBod3VzejdUR21OaE9lYjRPQ1F2RjQwMEl6Z05aSWJmdlhWVlBqMHhLeFU4dFBQCk5XT1VnN2ZTbjg5OUFmTmU1bmt5cWw3bWU4SVNQb0ozR1FLQ0FRQlBhOVZwM0pkOE9CZUlQelhsc2lPS2haSFIKYS94a1BqbnBVMkJvZzAzYllEcXdjWFRnOWtOWTc2U3FteXNaSFlkTDFCc2NLUFplZzc2SS9hbXhxYTZib29wWgpzTHAzOEQ0dk1NNk9ZeVhzaC9zUk82NDVuQjhIMGpzcGN4eXhySzF1MlBiK3M0MWZXTGVGNC9NWWdiMFFtbFFDCkYyZEJwVTkvZHJSQjhRNmd6NnFHeFQ5THk0SUxnazFhU05HMTRTTCtQWGg5amlPem92c05IY1FSbWI1eldJVkgKbURrdGdoMFh0S3NKVStBbEtrMHVoaUxZeGZNd3U3ZDhreENEdmhGYmRUQ3BIWjdNbzEzdW1udmNJRjllbnRoZApRc1BiYm5TK2JFVHZYckVWajA3R01rU0FJcjBkY0ZONldyTjRuTTVDcmpmMk1NQmJTSDFnR2J6QjZmTzEKLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K", + "external_public_key": "", + "enable_jwt_provider": true, + "jwk_url": "", + "jwt_user_path": "email", + "issuer_id": "d515ef82-5579-44d0-9919-59d273dca7c0" + }, + "websocket_config": { + "write_wait": 10, + "pong_wait": 60 + } + }, + { + "app_name": "integration-test", + "debug_message": true, + "tombstone_ttl": 180, + "auth_config": { + "init_user": "test", + "init_password": "test", + "internal_public_key": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQ0lqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FnOEFNSUlDQ2dLQ0FnRUE3TlJVL2FNWnF6Sm8vdnEyREFwMgpyenlWY1RPVHh0Mnh3VitnUUdDaGNWWlpsNVhDZk9FU1NEek5oZG1jTURnWUZ6MVpFd2loVUFDRDFKMzdXM0xRCkpWVzZVdHN1YTRMZG1COXgzci9teWV3OUMrRWZFNlBLY2pYSHU0NVBISTlRb1JDd1VzcDdGNE5PZElIWm12UjYKeFU0SkR0M09hcksvazVyaVpVVEd6WTJWbGxLUDdaL28wcXBGTmRDa2k3aDV6Vjd1TmlqMFFHbU5rN0hRa3FGMQpWSnIvU01oRStnTitUNmJiWVYrTUtqOWdtcmlRUDhyT0hYQUFjYkt0THRpSzlOM2FVWVhkM2Y0VnBWR0FaSlExCms1KzdvTWl5V2cxeTVGWWhFL1FyTTN3NlFtaWthUHAzci9LNUVXNUg2VUVCcEs2elRibU1JYUFJcVdMUnZ3ZjIKN3JFLy9mb0xhQ2ZEa1dFUGl5bGluKzFWOU4yM3JCMCtCWW1ta2VPcUt1OVpEdlUyV292VnRGeFdVQmU1TXNqSApPN2hGYzErUHEzcWU3bVZPNlVTdEdueUhJVEJjWVgrR3A3Qmo5amtrTEpoRS9OTHZ0akFQOFpQR1lCWXhZTHhNCjNndm9pN2RaaDFGU00vdUpGRjkzblRNNU0xWG1yR1MxaDZSbEFITE1ISUtSenZiYVRPRFZFdGtzUDNSQVUvVzEKZVZRYTVJUUd3a3FsZzdiNmYwcjNOZEcxbmhBWE1RZWo2QnZlOGIzZWozaHozQ2ZydnUwNnNFcjJqREF0UXZSNApZdGdUcDZ1Z1lxOVFqdENuRmR4RTNrdlVqNWxNMW1VODJhL09aYjRyZklISGFQTjR0TGJTZW1LZWJVZGliVzd0ClFWTktuMzNVLzdpbW5TQ2xQQ3ZHbGVVQ0F3RUFBUT09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo=", + "internal_private_key": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS1FJQkFBS0NBZ0VBN05SVS9hTVpxekpvL3ZxMkRBcDJyenlWY1RPVHh0Mnh3VitnUUdDaGNWWlpsNVhDCmZPRVNTRHpOaGRtY01EZ1lGejFaRXdpaFVBQ0QxSjM3VzNMUUpWVzZVdHN1YTRMZG1COXgzci9teWV3OUMrRWYKRTZQS2NqWEh1NDVQSEk5UW9SQ3dVc3A3RjROT2RJSFptdlI2eFU0SkR0M09hcksvazVyaVpVVEd6WTJWbGxLUAo3Wi9vMHFwRk5kQ2tpN2g1elY3dU5pajBRR21OazdIUWtxRjFWSnIvU01oRStnTitUNmJiWVYrTUtqOWdtcmlRClA4ck9IWEFBY2JLdEx0aUs5TjNhVVlYZDNmNFZwVkdBWkpRMWs1KzdvTWl5V2cxeTVGWWhFL1FyTTN3NlFtaWsKYVBwM3IvSzVFVzVINlVFQnBLNnpUYm1NSWFBSXFXTFJ2d2YyN3JFLy9mb0xhQ2ZEa1dFUGl5bGluKzFWOU4yMwpyQjArQlltbWtlT3FLdTlaRHZVMldvdlZ0RnhXVUJlNU1zakhPN2hGYzErUHEzcWU3bVZPNlVTdEdueUhJVEJjCllYK0dwN0JqOWpra0xKaEUvTkx2dGpBUDhaUEdZQll4WUx4TTNndm9pN2RaaDFGU00vdUpGRjkzblRNNU0xWG0KckdTMWg2UmxBSExNSElLUnp2YmFUT0RWRXRrc1AzUkFVL1cxZVZRYTVJUUd3a3FsZzdiNmYwcjNOZEcxbmhBWApNUWVqNkJ2ZThiM2VqM2h6M0NmcnZ1MDZzRXIyakRBdFF2UjRZdGdUcDZ1Z1lxOVFqdENuRmR4RTNrdlVqNWxNCjFtVTgyYS9PWmI0cmZJSEhhUE40dExiU2VtS2ViVWRpYlc3dFFWTktuMzNVLzdpbW5TQ2xQQ3ZHbGVVQ0F3RUEKQVFLQ0FnRUF4YnlYWmRxTE1ReDY3QUhxZyswc29TMkZYU29DUmJXT2wvVSt1T0cxaWdyZDdSbkdkRHY3NXNLVgpteDlSTUZWMWo5blNDSGxaTHBIdmdGT1RyZ3dUekVoaXRKRjZsWnVEWjJOQjBRa0xLaWNMNVdKR2IwQi9aSktRCnZJR2FmaThPMUJ3NkREWXhSaldGQ1BQdCsxb0xNN2Z2Q2pHYUNpNUtsSFJxZU1GTytBc3lEWHZMM2t4NHVZUWYKRzBxa1NHQnpta3lidWk3Qm1SSkllanVwK1BQRUlpcno0UklOdlcyelJjLzhOYlh3TXNvTjM4RWY5NU5lT3VmcwpCd0ozWkxpNmRLN1RmT08zbG9WeUQwRVlZV0g1eGRORmRuNTdvNEs5SGZibjBXQTV2ZGdJVUxCTUxiYUt2aVo3CjdSaldBK1FaK2lVL1lqTDgxSXBwRVB5SVFlYmxmVlRqQitQa3pKSWQ1R2dac0x6MlFka09xRnZWVE9pT0hNYlQKbDBIS3RmaFpod2pBRG1FeVdQaXIxOHNuVGs3SHoxbmh0eHV3ZGRlREZsZFZ2cU1xZlEyK25tTVBYRjdXbk9uWAppTGcyQlE0ZDFHS3c0WTBMR3FPdWl3ckFtOThlMEJLN2swdTN5WlVtRzRQRE91bzZFV2pyM0pCQ1p1YmViSVBvCkhhVnpBWllhTlZsbHN2dm9LOGp6eWN5aHo3cTBjK1lwRUJHLzNHTGFFbmdaSzVMUzRTZnhxU1FXc25aZjBmeE0KL0VBdWpEMUVjanZaTXJ6ejZ3SWxkd2I3Qkh6N3NBb1dXdmh1aUdZcEdWSzZSTUZhUDNmdXFvMkkwRVUrdEV5QQpOaW5mVDA3SkFGNnUwT1Zpc0dndVdiMHNGR2VKWis3UlVTc3RUbkYxQWh6dTgzdUM5N1VDZ2dFQkFQZzl4MXJqCjl5ZGhEaGFYWXQxNk9sQWU5NFR6eW41N2NzUlBlRlRSZjV4ZjF1MHVyQzJYZ2IvYnZ3MHI2SGc3bVBIRUV1NjMKU1ZxNUx2QzFEVHo5QzNJR1o3MkpyM1RWVTRsNlI2clVtNVBUYXExaE10M25WWlVLK2tUa0NWNE5wOVE3Vm0wMgpWT1IrSjZpM2JZOFU1bkZEemU1ZEk2QjRzWTFoeGQweW1wQW5Xc3dPSG5HMnFlclVEaHFHU2lJOFpzNmo0NTZJCmJZSkFkZ1BQRVZHNVY1V3owbnp2RG1YK3RQM05RRXF3NEE2MW9tQW9tVS8zODNQSVJ4MnJJSjRTT3JlUXNyd1QKdXBHTVJ0QWZ4bVV1aVJvRHI2eFNiczJ6SWRZcVgwcloxSnlBaXdqTUpNa2RBWWZmMGw1UUlwK1VVOTNYQ01nawpuSmFoc3JzRDdpcVluanNDZ2dFQkFQUTdQcVFTay9wY2ltdS9LSVhRSHhEN0N2Q2txY3I4UGpWTVo0NG5jMk5vCkZROEU2UzgxQmNFVEVDWmx4MVRyVkFVVmdFajhwTG9rTytHcUlzMGtHMHduQUNITVZLMS9nbE4vRCtSbm9zYXYKR1hmbnpnRGJzVVlNcDRCcDAwK1Uyc1ByWGFyOFU3ZWRZSnM2ZmhKcTAvbC85Y3BHcTByUHlncVZiWUd5KzBsaQpiR2NkMkxvTEhOdmdRS2dLZys1Zmg2WHFMU3lXWnJLcWQxVnhmSGVyZ01GUis4RXhodXNXRlpLTzFHZ1FPMzc2CjRsV0ZuUDBtMnJLMjlzSDJ4bWE4UVE0STFjRS95VEhBTDNOTUoyTCtTQjlHUy91ZGc5WGczdFI4dytZdW1EN3EKR09GeWdtTmZzR1NYYXpkRWxBTEFHRElENUEwd0UvRWdselNCanpxM3VsOENnZ0VBWTE2SGdMQ2tiTlVEQ0xRTQoxVTlxTEV4WkZKVnFSM3N2RTdva0Z2L05yMUVGL2VlaThKVW5VUitydUtBTTdLUWVzeGlqNDM3bkZEUHd3RllaCk9JS3FwRGhBS3JVRTBTWGJ6THB3R2NnRmh3VW9QTU1kMDRvWXpoS1k0QjdRU1IvNlFKQ0lKaXVMaS9PYitJT0UKamJQMkV2enJZREZVWTVZc3JNV29xTVRxN2kxeXdTQWR1N005RFUxWlgvREZtRExKaklvNlFXbW5QRzZGVHowQwpWODV6YXUrU29JUXBKVmJ5S0c2Uy85TVJ2Wkdqc0E1UVlKeUdqYUJzSjBvclFsdFZ1Y2xvWXJVYkI4dzVSSEtUCnZra0VoSzlaRVFmbVp0ei8vSFQxdEViQ1B1dU52RFhMdTkycWtUTmRTSGVYaEgyaG5MbkpRQ1Mzc2V5RVdTeFgKbUNHRHBRS0NBUUVBaTNyYVIzR2t1VExvaXFoZFNDNlh6MmJQMUtiMW9VdDFhNUw3QVNCZXNjTGJZL3gxLzlQVQpPWFBkb1ZBM0NyUnJBNHhIKzJidDNMQ2Mwa0FNS0FRYTR0N1RJSHBGVWVDa1dYTVRiR29UZUV5L3lzN0R3NUcwCktFRkoxL2lZQ2JjRlNTYStFOHlQTXluWjVrejlleDh2ZUNvd0FSbGk4aExCWEZJQ2ZEUHZkdldTMjBFY2FRTzMKczRyYTRoMC9RMytqUkluOHlwNEtnTGNCOS9ZY0Uyd0syRjB0M2lPZTNkdDY3bnhMcWpLN0I4WFlST2ROeFBYUApxSWo5VzhESGhoeTFPb0twTVBod3VzejdUR21OaE9lYjRPQ1F2RjQwMEl6Z05aSWJmdlhWVlBqMHhLeFU4dFBQCk5XT1VnN2ZTbjg5OUFmTmU1bmt5cWw3bWU4SVNQb0ozR1FLQ0FRQlBhOVZwM0pkOE9CZUlQelhsc2lPS2haSFIKYS94a1BqbnBVMkJvZzAzYllEcXdjWFRnOWtOWTc2U3FteXNaSFlkTDFCc2NLUFplZzc2SS9hbXhxYTZib29wWgpzTHAzOEQ0dk1NNk9ZeVhzaC9zUk82NDVuQjhIMGpzcGN4eXhySzF1MlBiK3M0MWZXTGVGNC9NWWdiMFFtbFFDCkYyZEJwVTkvZHJSQjhRNmd6NnFHeFQ5THk0SUxnazFhU05HMTRTTCtQWGg5amlPem92c05IY1FSbWI1eldJVkgKbURrdGdoMFh0S3NKVStBbEtrMHVoaUxZeGZNd3U3ZDhreENEdmhGYmRUQ3BIWjdNbzEzdW1udmNJRjllbnRoZApRc1BiYm5TK2JFVHZYckVWajA3R01rU0FJcjBkY0ZONldyTjRuTTVDcmpmMk1NQmJTSDFnR2J6QjZmTzEKLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K", + "external_public_key": "", + "enable_jwt_provider": true, + "jwk_url": "", + "jwt_user_path": "email", + "issuer_id": "d515ef82-5579-44d0-9919-59d273dca7c0" + }, + "websocket_config": { + "write_wait": 10, + "pong_wait": 60 + } + }, + { + "app_name": "junit-test", + "debug_message": true, + "tombstone_ttl": 180, + "auth_config": { + "init_user": "test", + "init_password": "test", + "internal_public_key": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQ0lqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FnOEFNSUlDQ2dLQ0FnRUE3TlJVL2FNWnF6Sm8vdnEyREFwMgpyenlWY1RPVHh0Mnh3VitnUUdDaGNWWlpsNVhDZk9FU1NEek5oZG1jTURnWUZ6MVpFd2loVUFDRDFKMzdXM0xRCkpWVzZVdHN1YTRMZG1COXgzci9teWV3OUMrRWZFNlBLY2pYSHU0NVBISTlRb1JDd1VzcDdGNE5PZElIWm12UjYKeFU0SkR0M09hcksvazVyaVpVVEd6WTJWbGxLUDdaL28wcXBGTmRDa2k3aDV6Vjd1TmlqMFFHbU5rN0hRa3FGMQpWSnIvU01oRStnTitUNmJiWVYrTUtqOWdtcmlRUDhyT0hYQUFjYkt0THRpSzlOM2FVWVhkM2Y0VnBWR0FaSlExCms1KzdvTWl5V2cxeTVGWWhFL1FyTTN3NlFtaWthUHAzci9LNUVXNUg2VUVCcEs2elRibU1JYUFJcVdMUnZ3ZjIKN3JFLy9mb0xhQ2ZEa1dFUGl5bGluKzFWOU4yM3JCMCtCWW1ta2VPcUt1OVpEdlUyV292VnRGeFdVQmU1TXNqSApPN2hGYzErUHEzcWU3bVZPNlVTdEdueUhJVEJjWVgrR3A3Qmo5amtrTEpoRS9OTHZ0akFQOFpQR1lCWXhZTHhNCjNndm9pN2RaaDFGU00vdUpGRjkzblRNNU0xWG1yR1MxaDZSbEFITE1ISUtSenZiYVRPRFZFdGtzUDNSQVUvVzEKZVZRYTVJUUd3a3FsZzdiNmYwcjNOZEcxbmhBWE1RZWo2QnZlOGIzZWozaHozQ2ZydnUwNnNFcjJqREF0UXZSNApZdGdUcDZ1Z1lxOVFqdENuRmR4RTNrdlVqNWxNMW1VODJhL09aYjRyZklISGFQTjR0TGJTZW1LZWJVZGliVzd0ClFWTktuMzNVLzdpbW5TQ2xQQ3ZHbGVVQ0F3RUFBUT09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo=", + "internal_private_key": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS1FJQkFBS0NBZ0VBN05SVS9hTVpxekpvL3ZxMkRBcDJyenlWY1RPVHh0Mnh3VitnUUdDaGNWWlpsNVhDCmZPRVNTRHpOaGRtY01EZ1lGejFaRXdpaFVBQ0QxSjM3VzNMUUpWVzZVdHN1YTRMZG1COXgzci9teWV3OUMrRWYKRTZQS2NqWEh1NDVQSEk5UW9SQ3dVc3A3RjROT2RJSFptdlI2eFU0SkR0M09hcksvazVyaVpVVEd6WTJWbGxLUAo3Wi9vMHFwRk5kQ2tpN2g1elY3dU5pajBRR21OazdIUWtxRjFWSnIvU01oRStnTitUNmJiWVYrTUtqOWdtcmlRClA4ck9IWEFBY2JLdEx0aUs5TjNhVVlYZDNmNFZwVkdBWkpRMWs1KzdvTWl5V2cxeTVGWWhFL1FyTTN3NlFtaWsKYVBwM3IvSzVFVzVINlVFQnBLNnpUYm1NSWFBSXFXTFJ2d2YyN3JFLy9mb0xhQ2ZEa1dFUGl5bGluKzFWOU4yMwpyQjArQlltbWtlT3FLdTlaRHZVMldvdlZ0RnhXVUJlNU1zakhPN2hGYzErUHEzcWU3bVZPNlVTdEdueUhJVEJjCllYK0dwN0JqOWpra0xKaEUvTkx2dGpBUDhaUEdZQll4WUx4TTNndm9pN2RaaDFGU00vdUpGRjkzblRNNU0xWG0KckdTMWg2UmxBSExNSElLUnp2YmFUT0RWRXRrc1AzUkFVL1cxZVZRYTVJUUd3a3FsZzdiNmYwcjNOZEcxbmhBWApNUWVqNkJ2ZThiM2VqM2h6M0NmcnZ1MDZzRXIyakRBdFF2UjRZdGdUcDZ1Z1lxOVFqdENuRmR4RTNrdlVqNWxNCjFtVTgyYS9PWmI0cmZJSEhhUE40dExiU2VtS2ViVWRpYlc3dFFWTktuMzNVLzdpbW5TQ2xQQ3ZHbGVVQ0F3RUEKQVFLQ0FnRUF4YnlYWmRxTE1ReDY3QUhxZyswc29TMkZYU29DUmJXT2wvVSt1T0cxaWdyZDdSbkdkRHY3NXNLVgpteDlSTUZWMWo5blNDSGxaTHBIdmdGT1RyZ3dUekVoaXRKRjZsWnVEWjJOQjBRa0xLaWNMNVdKR2IwQi9aSktRCnZJR2FmaThPMUJ3NkREWXhSaldGQ1BQdCsxb0xNN2Z2Q2pHYUNpNUtsSFJxZU1GTytBc3lEWHZMM2t4NHVZUWYKRzBxa1NHQnpta3lidWk3Qm1SSkllanVwK1BQRUlpcno0UklOdlcyelJjLzhOYlh3TXNvTjM4RWY5NU5lT3VmcwpCd0ozWkxpNmRLN1RmT08zbG9WeUQwRVlZV0g1eGRORmRuNTdvNEs5SGZibjBXQTV2ZGdJVUxCTUxiYUt2aVo3CjdSaldBK1FaK2lVL1lqTDgxSXBwRVB5SVFlYmxmVlRqQitQa3pKSWQ1R2dac0x6MlFka09xRnZWVE9pT0hNYlQKbDBIS3RmaFpod2pBRG1FeVdQaXIxOHNuVGs3SHoxbmh0eHV3ZGRlREZsZFZ2cU1xZlEyK25tTVBYRjdXbk9uWAppTGcyQlE0ZDFHS3c0WTBMR3FPdWl3ckFtOThlMEJLN2swdTN5WlVtRzRQRE91bzZFV2pyM0pCQ1p1YmViSVBvCkhhVnpBWllhTlZsbHN2dm9LOGp6eWN5aHo3cTBjK1lwRUJHLzNHTGFFbmdaSzVMUzRTZnhxU1FXc25aZjBmeE0KL0VBdWpEMUVjanZaTXJ6ejZ3SWxkd2I3Qkh6N3NBb1dXdmh1aUdZcEdWSzZSTUZhUDNmdXFvMkkwRVUrdEV5QQpOaW5mVDA3SkFGNnUwT1Zpc0dndVdiMHNGR2VKWis3UlVTc3RUbkYxQWh6dTgzdUM5N1VDZ2dFQkFQZzl4MXJqCjl5ZGhEaGFYWXQxNk9sQWU5NFR6eW41N2NzUlBlRlRSZjV4ZjF1MHVyQzJYZ2IvYnZ3MHI2SGc3bVBIRUV1NjMKU1ZxNUx2QzFEVHo5QzNJR1o3MkpyM1RWVTRsNlI2clVtNVBUYXExaE10M25WWlVLK2tUa0NWNE5wOVE3Vm0wMgpWT1IrSjZpM2JZOFU1bkZEemU1ZEk2QjRzWTFoeGQweW1wQW5Xc3dPSG5HMnFlclVEaHFHU2lJOFpzNmo0NTZJCmJZSkFkZ1BQRVZHNVY1V3owbnp2RG1YK3RQM05RRXF3NEE2MW9tQW9tVS8zODNQSVJ4MnJJSjRTT3JlUXNyd1QKdXBHTVJ0QWZ4bVV1aVJvRHI2eFNiczJ6SWRZcVgwcloxSnlBaXdqTUpNa2RBWWZmMGw1UUlwK1VVOTNYQ01nawpuSmFoc3JzRDdpcVluanNDZ2dFQkFQUTdQcVFTay9wY2ltdS9LSVhRSHhEN0N2Q2txY3I4UGpWTVo0NG5jMk5vCkZROEU2UzgxQmNFVEVDWmx4MVRyVkFVVmdFajhwTG9rTytHcUlzMGtHMHduQUNITVZLMS9nbE4vRCtSbm9zYXYKR1hmbnpnRGJzVVlNcDRCcDAwK1Uyc1ByWGFyOFU3ZWRZSnM2ZmhKcTAvbC85Y3BHcTByUHlncVZiWUd5KzBsaQpiR2NkMkxvTEhOdmdRS2dLZys1Zmg2WHFMU3lXWnJLcWQxVnhmSGVyZ01GUis4RXhodXNXRlpLTzFHZ1FPMzc2CjRsV0ZuUDBtMnJLMjlzSDJ4bWE4UVE0STFjRS95VEhBTDNOTUoyTCtTQjlHUy91ZGc5WGczdFI4dytZdW1EN3EKR09GeWdtTmZzR1NYYXpkRWxBTEFHRElENUEwd0UvRWdselNCanpxM3VsOENnZ0VBWTE2SGdMQ2tiTlVEQ0xRTQoxVTlxTEV4WkZKVnFSM3N2RTdva0Z2L05yMUVGL2VlaThKVW5VUitydUtBTTdLUWVzeGlqNDM3bkZEUHd3RllaCk9JS3FwRGhBS3JVRTBTWGJ6THB3R2NnRmh3VW9QTU1kMDRvWXpoS1k0QjdRU1IvNlFKQ0lKaXVMaS9PYitJT0UKamJQMkV2enJZREZVWTVZc3JNV29xTVRxN2kxeXdTQWR1N005RFUxWlgvREZtRExKaklvNlFXbW5QRzZGVHowQwpWODV6YXUrU29JUXBKVmJ5S0c2Uy85TVJ2Wkdqc0E1UVlKeUdqYUJzSjBvclFsdFZ1Y2xvWXJVYkI4dzVSSEtUCnZra0VoSzlaRVFmbVp0ei8vSFQxdEViQ1B1dU52RFhMdTkycWtUTmRTSGVYaEgyaG5MbkpRQ1Mzc2V5RVdTeFgKbUNHRHBRS0NBUUVBaTNyYVIzR2t1VExvaXFoZFNDNlh6MmJQMUtiMW9VdDFhNUw3QVNCZXNjTGJZL3gxLzlQVQpPWFBkb1ZBM0NyUnJBNHhIKzJidDNMQ2Mwa0FNS0FRYTR0N1RJSHBGVWVDa1dYTVRiR29UZUV5L3lzN0R3NUcwCktFRkoxL2lZQ2JjRlNTYStFOHlQTXluWjVrejlleDh2ZUNvd0FSbGk4aExCWEZJQ2ZEUHZkdldTMjBFY2FRTzMKczRyYTRoMC9RMytqUkluOHlwNEtnTGNCOS9ZY0Uyd0syRjB0M2lPZTNkdDY3bnhMcWpLN0I4WFlST2ROeFBYUApxSWo5VzhESGhoeTFPb0twTVBod3VzejdUR21OaE9lYjRPQ1F2RjQwMEl6Z05aSWJmdlhWVlBqMHhLeFU4dFBQCk5XT1VnN2ZTbjg5OUFmTmU1bmt5cWw3bWU4SVNQb0ozR1FLQ0FRQlBhOVZwM0pkOE9CZUlQelhsc2lPS2haSFIKYS94a1BqbnBVMkJvZzAzYllEcXdjWFRnOWtOWTc2U3FteXNaSFlkTDFCc2NLUFplZzc2SS9hbXhxYTZib29wWgpzTHAzOEQ0dk1NNk9ZeVhzaC9zUk82NDVuQjhIMGpzcGN4eXhySzF1MlBiK3M0MWZXTGVGNC9NWWdiMFFtbFFDCkYyZEJwVTkvZHJSQjhRNmd6NnFHeFQ5THk0SUxnazFhU05HMTRTTCtQWGg5amlPem92c05IY1FSbWI1eldJVkgKbURrdGdoMFh0S3NKVStBbEtrMHVoaUxZeGZNd3U3ZDhreENEdmhGYmRUQ3BIWjdNbzEzdW1udmNJRjllbnRoZApRc1BiYm5TK2JFVHZYckVWajA3R01rU0FJcjBkY0ZONldyTjRuTTVDcmpmMk1NQmJTSDFnR2J6QjZmTzEKLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K", + "external_public_key": "", + "enable_jwt_provider": true, + "jwk_url": "", + "jwt_user_path": "email", + "issuer_id": "d515ef82-5579-44d0-9919-59d273dca7c0" + }, + "websocket_config": { + "write_wait": 10, + "pong_wait": 60 + } + } +] \ No newline at end of file diff --git a/nitrite-replication/src/test/resources/mongo-seed/serverConfig.json b/nitrite-replication/src/test/resources/mongo-seed/serverConfig.json new file mode 100644 index 000000000..5d5ca6c68 --- /dev/null +++ b/nitrite-replication/src/test/resources/mongo-seed/serverConfig.json @@ -0,0 +1,29 @@ +[ + { + "_id": "serverConfig", + "alert_config": { + "log_format": "text", + "response_time_threshold": 20, + "sentry_dsn": "", + "enable_syslog": false, + "syslog_address": "", + "slack_channel": "", + "email_address": "", + "air_brake_id": -1, + "air_brake_key": "", + "hide_sensitive_data": true + }, + "email_config": { + "smtp_host": "smtp.gmail.com", + "smtp_port": "587", + "auth_required": true, + "sender": "", + "password": "" + }, + "tls_config": { + "enable_ssl": false, + "server_key": "", + "server_cert": "" + } + } +] \ No newline at end of file diff --git a/nitrite-rocksdb-adapter/build.gradle b/nitrite-rocksdb-adapter/build.gradle index cbe151413..6b3d6318b 100644 --- a/nitrite-rocksdb-adapter/build.gradle +++ b/nitrite-rocksdb-adapter/build.gradle @@ -34,26 +34,28 @@ dependencies { exclude group: "org.objenesis" } - annotationProcessor "org.projectlombok:lombok:1.18.22" + annotationProcessor "org.projectlombok:lombok:1.18.24" - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" - testImplementation "uk.co.jemos.podam:podam:7.2.7.RELEASE" - testImplementation "com.github.javafaker:javafaker:1.0.2" + testAnnotationProcessor "org.projectlombok:lombok:1.18.24" + testImplementation "uk.co.jemos.podam:podam:7.2.9.RELEASE" + testImplementation ("com.github.javafaker:javafaker:1.0.2") { + exclude module: 'snakeyaml' + } testImplementation "junit:junit:4.13.2" - testImplementation "org.mockito:mockito-core:4.1.0" - testImplementation "org.apache.lucene:lucene-core:8.11.0" - testImplementation "org.apache.lucene:lucene-analyzers-common:8.11.0" - testImplementation "org.apache.lucene:lucene-queryparser:8.11.0" - testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.14.1" - testImplementation "org.apache.logging.log4j:log4j-core:2.15.0" - testImplementation "org.awaitility:awaitility:4.1.1" - testImplementation "joda-time:joda-time:2.10.13" + testImplementation "org.mockito:mockito-core:4.6.1" + testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2" + testImplementation "org.apache.logging.log4j:log4j-core:2.17.2" + testImplementation "org.awaitility:awaitility:4.2.0" + testImplementation "joda-time:joda-time:2.10.14" testImplementation "org.meanbean:meanbean:2.0.3" - testImplementation "com.fasterxml.jackson.core:jackson-databind:2.13.0" + testImplementation ("com.fasterxml.jackson.core:jackson-databind:2.13.3") { + exclude module: 'org.yaml' + } testImplementation "commons-io:commons-io:2.11.0" - testImplementation "com.google.guava:guava:31.0.1-jre" - testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:3.0.1" - testImplementation "com.sun.xml.bind:jaxb-impl:3.0.2" + testImplementation 'com.google.guava:guava:31.1-jre' + testImplementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0' + testImplementation 'com.sun.xml.bind:jaxb-impl:4.0.0' + testImplementation 'org.yaml:snakeyaml:1.30' } test { diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/EntrySet.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/EntrySet.java index 764952621..4b22bb002 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/EntrySet.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/EntrySet.java @@ -59,24 +59,13 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public Pair next() { K key = (K) objectFormatter.decodeKey(rawEntryIterator.key(), keyType); - try { - V value = (V) objectFormatter.decode(rawEntryIterator.value(), valueType); - if (reverse) { - rawEntryIterator.prev(); - } else { - rawEntryIterator.next(); - } - return new Pair<>(key, value); - } catch (Exception e) { - System.out.println(new String(rawEntryIterator.value())); - throw e; + V value = (V) objectFormatter.decode(rawEntryIterator.value(), valueType); + if (reverse) { + rawEntryIterator.prev(); + } else { + rawEntryIterator.next(); } - } - - @Override - protected void finalize() throws Throwable { - rawEntryIterator.close(); - super.finalize(); + return new Pair<>(key, value); } } } diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBMap.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBMap.java index 0f045fab0..fc403be9c 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBMap.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBMap.java @@ -64,7 +64,7 @@ public boolean containsKey(K k) { return rocksDB.get(columnFamilyHandle, key) != null; } catch (Exception e) { log.error("Error while querying key", e); - throw new NitriteIOException("failed to check key", e); + throw new NitriteIOException("Failed to check key", e); } } @@ -81,7 +81,7 @@ public V get(K k) { return (V) objectFormatter.decode(value, getValueType()); } catch (Exception e) { log.error("Error while querying by key", e); - throw new NitriteIOException("failed to query by key", e); + throw new NitriteIOException("Failed to query by key", e); } } @@ -134,7 +134,7 @@ public V remove(K k) { return (V) objectFormatter.decode(value, getValueType()); } catch (Exception e) { log.error("Error while removing key", e); - throw new NitriteIOException("failed to remove key", e); + throw new NitriteIOException("Failed to remove key", e); } } @@ -162,7 +162,7 @@ public void put(K k, V v) { updateLastModifiedTime(); } catch (Exception e) { log.error("Error while writing key and value for " + mapName, e); - throw new NitriteIOException("failed to write key and value", e); + throw new NitriteIOException("Failed to write key and value", e); } } @@ -204,7 +204,7 @@ public V putIfAbsent(K k, V v) { return (V) objectFormatter.decode(oldValue, getValueType()); } catch (Exception e) { log.error("Error while writing key and value", e); - throw new NitriteIOException("failed to write key and value", e); + throw new NitriteIOException("Failed to write key and value", e); } } @@ -344,6 +344,11 @@ public void drop() { } } + @Override + public boolean isDropped() { + return droppedFlag.get(); + } + @Override public void close() { if (!closedFlag.get() && !droppedFlag.get()) { @@ -352,6 +357,11 @@ public void close() { } } + @Override + public boolean isClosed() { + return closedFlag.get(); + } + private void initialize() { this.size = new AtomicLong(0); // just initialized this.closedFlag = new AtomicBoolean(false); diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBRTree.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBRTree.java new file mode 100644 index 000000000..0c2534059 --- /dev/null +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBRTree.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.rocksdb; + +import org.dizitart.no2.collection.NitriteId; +import org.dizitart.no2.common.RecordStream; +import org.dizitart.no2.common.util.SpatialKey; +import org.dizitart.no2.exceptions.InvalidOperationException; +import org.dizitart.no2.index.BoundingBox; +import org.dizitart.no2.store.NitriteRTree; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author Anindya Chatterjee + */ +public class RocksDBRTree implements NitriteRTree { + private final RocksDBMap backingMap; + private final AtomicBoolean droppedFlag; + private final AtomicBoolean closedFlag; + + public RocksDBRTree(RocksDBMap backingMap) { + this.backingMap = backingMap; + this.closedFlag = new AtomicBoolean(false); + this.droppedFlag = new AtomicBoolean(false); + } + + @Override + public void add(Key key, NitriteId nitriteId) { + checkOpened(); + if (nitriteId != null && nitriteId.getIdValue() != null) { + SpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue())); + backingMap.put(spatialKey, key); + } + } + + @Override + public void remove(Key key, NitriteId nitriteId) { + checkOpened(); + if (nitriteId != null && nitriteId.getIdValue() != null) { + SpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue())); + backingMap.remove(spatialKey); + } + } + + @Override + public RecordStream findIntersectingKeys(Key key) { + checkOpened(); + SpatialKey spatialKey = getKey(key, 0L); + Set set = new HashSet<>(); + + for (SpatialKey sk : backingMap.keys()) { + if (isOverlap(sk, spatialKey)) { + set.add(NitriteId.createId(Long.toString(sk.getId()))); + } + } + + return RecordStream.fromIterable(set); + } + + @Override + public RecordStream findContainedKeys(Key key) { + checkOpened(); + SpatialKey spatialKey = getKey(key, 0L); + Set set = new HashSet<>(); + + for (SpatialKey sk : backingMap.keys()) { + if (isInside(sk, spatialKey)) { + set.add(NitriteId.createId(Long.toString(sk.getId()))); + } + } + + return RecordStream.fromIterable(set); + } + + @Override + public long size() { + checkOpened(); + return backingMap.size(); + } + + @Override + public void close() { + closedFlag.compareAndSet(false, true); + } + + @Override + public void clear() { + checkOpened(); + backingMap.clear(); + } + + @Override + public void drop() { + checkOpened(); + droppedFlag.compareAndSet(false, true); + backingMap.clear(); + } + + private SpatialKey getKey(Key key, long id) { + return new SpatialKey(id, key.getMinX(), + key.getMaxX(), key.getMinY(), key.getMaxY()); + } + + private boolean isOverlap(SpatialKey a, SpatialKey b) { + if (a.isNull() || b.isNull()) { + return false; + } + for (int i = 0; i < 2; i++) { + if (a.max(i) < b.min(i) || a.min(i) > b.max(i)) { + return false; + } + } + return true; + } + + private boolean isInside(SpatialKey a, SpatialKey b) { + if (a.isNull() || b.isNull()) { + return false; + } + for (int i = 0; i < 2; i++) { + if (a.min(i) <= b.min(i) || a.max(i) >= b.max(i)) { + return false; + } + } + return true; + } + + private void checkOpened() { + if (closedFlag.get()) { + throw new InvalidOperationException("RTreeMap is closed"); + } + + if (droppedFlag.get()) { + throw new InvalidOperationException("RTreeMap is dropped"); + } + } +} diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBReference.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBReference.java index 8b1299e08..3bdd886e2 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBReference.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBReference.java @@ -75,7 +75,7 @@ public synchronized ColumnFamilyHandle getOrCreateColumnFamily(String name) { return handle; } catch (RocksDBException e) { log.error("Error while retrieving column family handle", e); - throw new NitriteIOException("failed to obtain column family handle", e); + throw new NitriteIOException("Failed to obtain column family handle", e); } } } @@ -89,7 +89,7 @@ public void dropColumnFamily(String mapName) { columnFamilyHandleRegistry.remove(mapName); } catch (RocksDBException e) { log.error("Error while dropping column family " + mapName, e); - throw new NitriteIOException("failed to drop column family", e); + throw new NitriteIOException("Failed to drop column family", e); } } } diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStore.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStore.java index 4199fbeb7..7cb929c91 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStore.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStore.java @@ -2,8 +2,8 @@ import lombok.extern.slf4j.Slf4j; import org.dizitart.no2.common.UnknownType; +import org.dizitart.no2.common.util.SpatialKey; import org.dizitart.no2.common.util.StringUtils; -import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.NitriteException; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.index.BoundingBox; @@ -22,11 +22,14 @@ public class RocksDBStore extends AbstractNitriteStore { private final AtomicBoolean closed; private final Map> nitriteMapRegistry; + + private final Map> nitriteRTreeMapRegistry; private RocksDBReference reference; public RocksDBStore() { super(); nitriteMapRegistry = new ConcurrentHashMap<>(); + nitriteRTreeMapRegistry = new ConcurrentHashMap<>(); closed = new AtomicBoolean(true); } @@ -43,7 +46,7 @@ public void openOrCreate() { throw e; } catch (Exception e) { log.error("Error while opening database", e); - throw new NitriteIOException("failed to open database", e); + throw new NitriteIOException("Failed to open database", e); } } @@ -84,7 +87,7 @@ public void close() { eventBus.close(); } catch (Exception e) { log.error("Error while closing the database", e); - throw new NitriteIOException("failed to close database", e); + throw new NitriteIOException("Failed to close database", e); } } @@ -131,20 +134,34 @@ public void removeMap(String mapName) { } @Override + @SuppressWarnings("unchecked") public NitriteRTree openRTree(String rTreeName, Class keyType, Class valueType) { - throw new InvalidOperationException("rtree not supported on rocksdb store"); + + if (nitriteRTreeMapRegistry.containsKey(rTreeName)) { + return (RocksDBRTree) nitriteRTreeMapRegistry.get(rTreeName); + } else { + RocksDBMap nitriteMap = new RocksDBMap<>(rTreeName, this, this.reference, + SpatialKey.class, keyType); + RocksDBRTree nitriteRTree = new RocksDBRTree<>(nitriteMap); + nitriteRTreeMapRegistry.put(rTreeName, nitriteRTree); + return nitriteRTree; + } } @Override public void closeRTree(String rTreeName) { - throw new InvalidOperationException("rtree not supported on rocksdb store"); + if (!StringUtils.isNullOrEmpty(rTreeName)) { + nitriteRTreeMapRegistry.remove(rTreeName); + } } @Override public void removeRTree(String mapName) { - throw new InvalidOperationException("rtree not supported on rocksdb store"); + reference.dropColumnFamily(mapName); + getCatalog().remove(mapName); + nitriteRTreeMapRegistry.remove(mapName); } @Override diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStoreUtils.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStoreUtils.java index 07f20b82e..7b071c64c 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStoreUtils.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/RocksDBStoreUtils.java @@ -15,7 +15,7 @@ public static RocksDBReference openOrCreate(RocksDBConfig storeConfig) { if (!isNullOrEmpty(storeConfig.filePath())) { db = StoreFactory.createDBReference(storeConfig); } else { - throw new InvalidOperationException("nitrite rocksdb store does not support in-memory database"); + throw new InvalidOperationException("Nitrite rocksdb store does not support in-memory database"); } return db; } diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/StoreFactory.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/StoreFactory.java index d292ff285..a7f86e175 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/StoreFactory.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/StoreFactory.java @@ -100,7 +100,7 @@ private static void createColumnFamilyDescriptors(RocksDBReference reference, Ro } } catch (RocksDBException e) { log.error("Error while listing column families", e); - throw new NitriteIOException("failed to open database", e); + throw new NitriteIOException("Failed to open database", e); } reference.setColumnFamilyDescriptors(cfDescriptors); } @@ -120,7 +120,7 @@ private static void createRocksDB(RocksDBReference reference, RocksDBConfig dbCo reference.setColumnFamilyHandleRegistry(handleMap); } catch (RocksDBException e) { log.error("Error while opening rocks database", e); - throw new NitriteIOException("failed to open database", e); + throw new NitriteIOException("Failed to open database", e); } } } diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/DefaultTimeKeySerializers.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/DefaultTimeKeySerializers.java index 0fa3b24ef..097817233 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/DefaultTimeKeySerializers.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/DefaultTimeKeySerializers.java @@ -30,7 +30,7 @@ public Date readKeyInternal(Kryo kryo, String value, Class type) { try { return format.parse(value); } catch (Exception e) { - throw new NitriteIOException("failed to read java.util.Date", e); + throw new NitriteIOException("Failed to read java.util.Date", e); } } } @@ -47,7 +47,7 @@ public Timestamp readKeyInternal(Kryo kryo, String value, Class type) try { return new Timestamp(format.parse(value).getTime()); } catch (Exception e) { - throw new NitriteIOException("failed to read java.sql.Timestamp", e); + throw new NitriteIOException("Failed to read java.sql.Timestamp", e); } } } @@ -64,7 +64,7 @@ public java.sql.Date readKeyInternal(Kryo kryo, String value, Class type) { try { return new Time(format.parse(value).getTime()); } catch (Exception e) { - throw new NitriteIOException("failed to read java.sql.Time", e); + throw new NitriteIOException("Failed to read java.sql.Time", e); } } } @@ -100,7 +100,7 @@ protected Calendar readKeyInternal(Kryo kryo, String input, Class type cal.setTime(format.parse(input)); return cal; } catch (Exception e) { - throw new NitriteIOException("failed to read java.util.Date", e); + throw new NitriteIOException("Failed to read java.util.Date", e); } } } diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/KryoObjectFormatter.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/KryoObjectFormatter.java index 28893f4db..29d1a392c 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/KryoObjectFormatter.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/KryoObjectFormatter.java @@ -59,7 +59,7 @@ public byte[] encode(T object) { } return byteArrayOutputStream.toByteArray(); } catch (IOException e) { - throw new NitriteIOException("failed to close output stream", e); + throw new NitriteIOException("Failed to close output stream", e); } } @@ -80,7 +80,7 @@ public byte[] encodeKey(T object) { } return byteArrayOutputStream.toByteArray(); } catch (IOException e) { - throw new NitriteIOException("failed to close output stream", e); + throw new NitriteIOException("Failed to close output stream", e); } } @@ -110,7 +110,7 @@ public T decodeKey(byte[] bytes, Class type) { return serializer.readKey(kryo, input, type); } } catch (IOException e) { - throw new NitriteIOException("failed to close output stream", e); + throw new NitriteIOException("Failed to close output stream", e); } } @@ -133,7 +133,7 @@ private void registerInternalSerializers() { DefaultTimeKeySerializers.registerAll(this); } catch (Exception e) { log.error("Error while registering default serializers", e); - throw new NitriteIOException("failed to register default serializers", e); + throw new NitriteIOException("Failed to register default serializers", e); } } } diff --git a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/NitriteSerializers.java b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/NitriteSerializers.java index 32e0267ef..d570520a8 100644 --- a/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/NitriteSerializers.java +++ b/nitrite-rocksdb-adapter/src/main/java/org/dizitart/no2/rocksdb/formatter/NitriteSerializers.java @@ -9,10 +9,10 @@ import com.esotericsoftware.kryo.kryo5.serializers.MapSerializer; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.common.Fields; import org.dizitart.no2.common.tuples.Pair; -import org.dizitart.no2.index.DBValue; +import org.dizitart.no2.common.DBValue; import org.dizitart.no2.index.IndexDescriptor; import org.dizitart.no2.index.IndexMeta; import org.dizitart.no2.store.UserCredential; @@ -115,7 +115,7 @@ private static class IndexDescriptorSerializer extends Serializer repositories = db.listRepositories(); assertEquals(repositories.size(), 1); } @@ -141,10 +153,15 @@ public void testHasCollection() { assertFalse(db.hasCollection("lucene" + INTERNAL_NAME_SEPARATOR + "test")); } - @Test + @Test(expected = ValidationException.class) public void testHasRepository() { db.getRepository(getClass()); - assertTrue(db.hasRepository(getClass())); + } + + @Test + public void testHasRepository2() { + db.getRepository(Receipt.class); + assertTrue(db.hasRepository(Receipt.class)); assertFalse(db.hasRepository(String.class)); } @@ -205,20 +222,30 @@ public void testGetCollection() { assertEquals(collection.getName(), "test-collection"); } - @Test + @Test(expected = ValidationException.class) public void testGetRepository() { - ObjectRepository repository = db.getRepository(NitriteTest.class); - assertNotNull(repository); - assertEquals(repository.getType(), NitriteTest.class); + ObjectRepository repository = db.getRepository(EmptyClass.class); } @Test + public void testGetRepository2() { + ObjectRepository repository = db.getRepository(Receipt.class); + assertNotNull(repository); + assertEquals(repository.getType(), Receipt.class); + } + + @Test(expected = ValidationException.class) public void testGetRepositoryWithKey() { - ObjectRepository repository = db.getRepository(NitriteTest.class, "key"); + ObjectRepository repository = db.getRepository(EmptyClass.class, "key"); + } + + @Test + public void testGetRepositoryWithKey2() { + ObjectRepository repository = db.getRepository(Receipt.class, "key"); assertNotNull(repository); - assertEquals(repository.getType(), NitriteTest.class); - assertFalse(db.hasRepository(NitriteTest.class)); - assertTrue(db.hasRepository(NitriteTest.class, "key")); + assertEquals(repository.getType(), Receipt.class); + assertFalse(db.hasRepository(Receipt.class)); + assertTrue(db.hasRepository(Receipt.class, "key")); } @Test @@ -234,18 +261,18 @@ public void testMultipleGetCollection() { @Test public void testMultipleGetRepository() { - ObjectRepository repository = db.getRepository(NitriteTest.class); + ObjectRepository repository = db.getRepository(Receipt.class); assertNotNull(repository); - assertEquals(repository.getType(), NitriteTest.class); + assertEquals(repository.getType(), Receipt.class); - ObjectRepository repository2 = db.getRepository(NitriteTest.class); + ObjectRepository repository2 = db.getRepository(Receipt.class); assertNotNull(repository2); - assertEquals(repository2.getType(), NitriteTest.class); + assertEquals(repository2.getType(), Receipt.class); } @Test(expected = ValidationException.class) public void testGetRepositoryInvalid() { - db.getRepository(null); + db.getRepository((Class) null); } @Test(expected = NitriteIOException.class) @@ -259,14 +286,14 @@ public void testGetCollectionNullStore() { public void testGetRepositoryNullStore() { db = Nitrite.builder().openOrCreate(); db.close(); - db.getRepository(NitriteTest.class); + db.getRepository(EmptyClass.class); } @Test(expected = NitriteIOException.class) public void testGetKeyedRepositoryNullStore() { db = Nitrite.builder().openOrCreate(); db.close(); - db.getRepository(NitriteTest.class, "key"); + db.getRepository(EmptyClass.class, "key"); } @Test(expected = NitriteIOException.class) @@ -331,11 +358,17 @@ public void testIssue193() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(10000); for (int i = 0; i < 10000; i++) { pool.submit(() -> { - int refIndex = random.nextInt(5); - Receipt receipt = factory.manufacturePojoWithFullData(Receipt.class); - receipt.setClientRef(refs[refIndex]); - repository.update(receipt, true); - latch.countDown(); + try { + int refIndex = random.nextInt(5); + Receipt receipt = factory.manufacturePojoWithFullData(Receipt.class); + receipt.setClientRef(refs[refIndex]); + repository.update(receipt, true); + } catch (Exception e) { + e.printStackTrace(); + fail("Unhandled exception in thread - " + e.getMessage()); + } finally { + latch.countDown(); + } }); } @@ -420,20 +453,30 @@ public void run() { @Data @AllArgsConstructor @NoArgsConstructor - public static class CompatChild implements Mappable { + public static class CompatChild { private Long childId; private String lastName; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("childId", childId) - .put("lastName", lastName); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - childId = document.get("childId", Long.class); - lastName = document.get("lastName", String.class); + @Override + public Class getEntityType() { + return CompatChild.class; + } + + @Override + public Document toDocument(CompatChild entity, NitriteMapper nitriteMapper) { + return Document.createDocument("childId", entity.childId) + .put("lastName", entity.lastName); + } + + @Override + public CompatChild fromDocument(Document document, NitriteMapper nitriteMapper) { + CompatChild entity = new CompatChild(); + entity.childId = document.get("childId", Long.class); + entity.lastName = document.get("lastName", String.class); + return entity; + } } } @@ -441,35 +484,47 @@ public void read(NitriteMapper mapper, Document document) { @NoArgsConstructor @AllArgsConstructor @Indices({ - @Index(value = "synced", type = IndexType.NON_UNIQUE) + @Index(fields = "synced", type = IndexType.NON_UNIQUE) }) - public static class Receipt implements Mappable { + public static class Receipt { @Id private String clientRef; private Boolean synced; private Status status; private Long createdTimestamp = System.currentTimeMillis(); - @Override - public Document write(NitriteMapper mapper) { - return createDocument("status", status) - .put("clientRef", clientRef) - .put("synced", synced) - .put("createdTimestamp", createdTimestamp); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return Receipt.class; + } - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - Object status = document.get("status"); - if (status instanceof Status) { - this.status = (Status) status; - } else { - this.status = Status.valueOf(status.toString()); + @Override + public Document toDocument(Receipt entity, NitriteMapper nitriteMapper) { + return createDocument("status", entity.status) + .put("clientRef", entity.clientRef) + .put("synced", entity.synced) + .put("createdTimestamp", entity.createdTimestamp); + } + + @Override + public Receipt fromDocument(Document document, NitriteMapper nitriteMapper) { + Receipt receipt = new Receipt(); + if (document != null) { + Object status = document.get("status"); + if (status != null) { + if (status instanceof Receipt.Status) { + receipt.status = (Receipt.Status) status; + } else { + receipt.status = Receipt.Status.valueOf(status.toString()); + } + } + receipt.clientRef = document.get("clientRef", String.class); + receipt.synced = document.get("synced", Boolean.class); + receipt.createdTimestamp = document.get("createdTimestamp", Long.class); } - this.clientRef = document.get("clientRef", String.class); - this.synced = document.get("synced", Boolean.class); - this.createdTimestamp = document.get("createdTimestamp", Long.class); + return receipt; } } @@ -478,4 +533,4 @@ public enum Status { PREPARING, } } -} +} \ No newline at end of file diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java index f6436ee6d..eeb05035e 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteBuilderTest.java @@ -27,8 +27,9 @@ import org.dizitart.no2.collection.NitriteId; import org.dizitart.no2.common.FieldValues; import org.dizitart.no2.common.Fields; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.NitriteSecurityException; @@ -174,6 +175,10 @@ public void testPopulateRepositories() { .fieldSeparator(".") .openOrCreate(); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TestObject.Converter()); + documentMapper.registerEntityConverter(new TestObject2.Converter()); + NitriteCollection collection = db.getCollection("test"); collection.insert(createDocument("id1", "value")); @@ -304,24 +309,14 @@ public Target convert(Source source, Class type) { return null; } - @Override - public boolean isValueType(Class type) { - return false; - } - - @Override - public boolean isValue(Object object) { - return false; - } - @Override public void initialize(NitriteConfig nitriteConfig) { } } - @Index(value = "longValue") - private static class TestObject implements Mappable { + @Index(fields = "longValue") + private static class TestObject { private String stringValue; private Long longValue; @@ -333,23 +328,32 @@ public TestObject(String stringValue, Long longValue) { this.stringValue = stringValue; } - @Override - public Document write(NitriteMapper mapper) { - return createDocument("stringValue", stringValue) - .put("longValue", longValue); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - this.stringValue = document.get("stringValue", String.class); - this.longValue = document.get("longValue", Long.class); + @Override + public Class getEntityType() { + return TestObject.class; + } + + @Override + public Document toDocument(TestObject entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("stringValue", entity.stringValue) + .put("longValue", entity.longValue); + } + + @Override + public TestObject fromDocument(Document document, NitriteMapper nitriteMapper) { + TestObject entity = new TestObject(); + entity.stringValue = document.get("stringValue", String.class); + entity.longValue = document.get("longValue", Long.class); + return entity; } } } - @Index(value = "longValue") - private static class TestObject2 implements Mappable { + @Index(fields = "longValue") + private static class TestObject2 { private String stringValue; private Long longValue; @@ -361,17 +365,27 @@ public TestObject2(String stringValue, Long longValue) { this.stringValue = stringValue; } - @Override - public Document write(NitriteMapper mapper) { - return createDocument("stringValue", stringValue) - .put("longValue", longValue); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - this.stringValue = document.get("stringValue", String.class); - this.longValue = document.get("longValue", Long.class); + @Override + public Class getEntityType() { + return TestObject2.class; + } + + @Override + public Document toDocument(TestObject2 entity, NitriteMapper nitriteMapper) { + return createDocument("stringValue", entity.stringValue) + .put("longValue", entity.longValue); + } + + @Override + public TestObject2 fromDocument(Document document, NitriteMapper nitriteMapper) { + TestObject2 entity = new TestObject2(); + if (document != null) { + entity.stringValue = document.get("stringValue", String.class); + entity.longValue = document.get("longValue", Long.class); + } + return entity; } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java index 222038b62..d4bef6baf 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteSecurityNegativeTest.java @@ -55,7 +55,7 @@ public void testOpenSecuredWithoutCredential() { assertEquals(dbCollection.find().size(), 1); } - @Test(expected = NitriteException.class) + @Test public void testOpenUnsecuredWithCredential() { db = createDb(fileName); NitriteCollection dbCollection = db.getCollection("test"); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java index 19c4a830a..ff2d6dd9d 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java @@ -24,8 +24,9 @@ import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.DocumentCursor; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.filters.Filter; import org.dizitart.no2.index.IndexOptions; import org.dizitart.no2.index.IndexType; @@ -66,6 +67,11 @@ public class NitriteStressTest { @Before public void before() { db = createDb(fileName); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TestDto.Converter()); + documentMapper.registerEntityConverter(new PerfTest.Converter()); + documentMapper.registerEntityConverter(new PerfTestIndexed.Converter()); + collection = db.getCollection("test"); System.out.println(fileName); } @@ -220,7 +226,7 @@ private List getItems(Class type) { } @Data - public static class TestDto implements Mappable { + public static class TestDto { @XmlElement( name = "StudentNumber", @@ -268,61 +274,108 @@ public static class TestDto implements Mappable { public TestDto() { } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("studentNumber", studentNumber) - .put("lastName", lastName) - .put("prefixes", prefixes) - .put("initials", initials) - .put("firstNames", firstNames) - .put("nickName", nickName) - .put("birthDate", birthDate); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return TestDto.class; + } + + @Override + public Document toDocument(TestDto entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("studentNumber", entity.studentNumber) + .put("lastName", entity.lastName) + .put("prefixes", entity.prefixes) + .put("initials", entity.initials) + .put("firstNames", entity.firstNames) + .put("nickName", entity.nickName) + .put("birthDate", entity.birthDate); + } - @Override - public void read(NitriteMapper mapper, Document document) { - studentNumber = document.get("studentNumber", String.class); - lastName = document.get("lastName", String.class); - prefixes = document.get("prefixes", String.class); - initials = document.get("initials", String.class); - firstNames = document.get("firstNames", String.class); - nickName = document.get("nickName", String.class); - birthDate = document.get("birthDate", String.class); + @Override + public TestDto fromDocument(Document document, NitriteMapper nitriteMapper) { + TestDto entity = new TestDto(); + entity.studentNumber = document.get("studentNumber", String.class); + entity.lastName = document.get("lastName", String.class); + entity.prefixes = document.get("prefixes", String.class); + entity.initials = document.get("initials", String.class); + entity.firstNames = document.get("firstNames", String.class); + entity.nickName = document.get("nickName", String.class); + entity.birthDate = document.get("birthDate", String.class); + return entity; + } } } @Data - public static class PerfTest implements Mappable { + public static class PerfTest { private String firstName; private String lastName; private Integer age; private String text; - @Override - public Document write(NitriteMapper mapper) { - Document document = Document.createDocument(); - document.put("firstName", firstName); - document.put("lastName", lastName); - document.put("age", age); - document.put("text", text); - return document; - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return PerfTest.class; + } - @Override - public void read(NitriteMapper mapper, Document document) { - this.firstName = (String) document.get("firstName"); - this.lastName = (String) document.get("lastName"); - this.age = (Integer) document.get("age"); - this.text = (String) document.get("text"); + @Override + public Document toDocument(PerfTest entity, NitriteMapper nitriteMapper) { + Document document = Document.createDocument(); + document.put("firstName", entity.firstName); + document.put("lastName", entity.lastName); + document.put("age", entity.age); + document.put("text", entity.text); + return document; + } + + @Override + public PerfTest fromDocument(Document document, NitriteMapper nitriteMapper) { + PerfTest entity = new PerfTest(); + entity.firstName = (String) document.get("firstName"); + entity.lastName = (String) document.get("lastName"); + entity.age = (Integer) document.get("age"); + entity.text = (String) document.get("text"); + return entity; + } } } @Indices({ - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "age", type = IndexType.NON_UNIQUE), - @Index(value = "text", type = IndexType.FULL_TEXT), + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "age", type = IndexType.NON_UNIQUE), + @Index(fields = "text", type = IndexType.FULL_TEXT), }) private static class PerfTestIndexed extends PerfTest { + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return PerfTestIndexed.class; + } + + @Override + public Document toDocument(PerfTestIndexed entity, NitriteMapper nitriteMapper) { + Document document = Document.createDocument(); + document.put("firstName", entity.getFirstName()); + document.put("lastName", entity.getLastName()); + document.put("age", entity.getAge()); + document.put("text", entity.getText()); + return document; + } + + @Override + public PerfTestIndexed fromDocument(Document document, NitriteMapper nitriteMapper) { + PerfTestIndexed entity = new PerfTestIndexed(); + entity.setFirstName((String) document.get("firstName")); + entity.setLastName((String) document.get("lastName")); + entity.setAge((Integer) document.get("age")); + entity.setText((String) document.get("text")); + return entity; + } + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java index f5396fdfe..20c06f02c 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/NitriteTest.java @@ -19,7 +19,9 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.repository.data.ClassA; +import org.dizitart.no2.integration.repository.data.ClassBConverter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -37,6 +39,10 @@ public class NitriteTest { @Before public void setUp() { db = createDb(fileName); + SimpleDocumentMapper nitriteMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + nitriteMapper.registerEntityConverter(new ClassA.ClassAConverter()); + nitriteMapper.registerEntityConverter(new ClassBConverter()); + NitriteCollection collection = db.getCollection("test"); assertNotNull(collection); } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/TestUtil.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/TestUtil.java index 91a4101ad..5d1689ef4 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/TestUtil.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/TestUtil.java @@ -124,7 +124,7 @@ public static Document parse(String json) { return loadDocument(node); } catch (IOException e) { log.error("Error while parsing json", e); - throw new ObjectMappingException("failed to parse json " + json); + throw new ObjectMappingException("Failed to parse json " + json); } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java index de25007a2..95243ae84 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java @@ -85,7 +85,7 @@ public void setUp() { .put("lastName", "ln2") .put("birthDay", simpleDateFormat.parse("2010-06-12T16:02:48.440Z")) .put("data", new byte[]{3, 4, 3}) - .put("list", Arrays.asList("three", "four", "three")) + .put("list", Arrays.asList("three", "four", "five")) .put("body", "quick hello world from nitrite"); doc3 = createDocument("firstName", "fn3") .put("lastName", "ln2") diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java index 8fd30693e..f1c1d927e 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java @@ -19,6 +19,7 @@ import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.DocumentCursor; +import org.dizitart.no2.collection.FindOptions; import org.dizitart.no2.collection.FindPlan; import org.dizitart.no2.common.SortOrder; import org.dizitart.no2.common.tuples.Pair; @@ -82,7 +83,7 @@ public void testFindByOrFilterAndFilter() { where("firstName").eq("fn3"), where("lastName").eq("ln2") ) - ) + ), FindOptions.withDistinct() ); assertEquals(2, cursor.size()); @@ -199,6 +200,22 @@ public void testFindByOrFilter() throws ParseException { FindPlan findPlan = cursor.getFindPlan(); assertEquals(3, findPlan.getSubPlans().size()); + assertEquals(5, cursor.size()); + + // distinct + cursor = collection.find( + or( + or( + where("lastName").eq("ln2"), + where("firstName").notEq("fn1") + ), + where("birthDay").eq(simpleDateFormat.parse("2012-07-01T16:02:48.440Z")), + where("firstName").notEq("fn1") + ), FindOptions.withDistinct() + ); + + findPlan = cursor.getFindPlan(); + assertEquals(3, findPlan.getSubPlans().size()); assertEquals(3, cursor.size()); } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java index 8d446cf0f..71a97a42d 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java @@ -21,6 +21,7 @@ import org.dizitart.no2.collection.DocumentCursor; import org.dizitart.no2.common.SortOrder; import org.dizitart.no2.exceptions.FilterException; +import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.ValidationException; import org.junit.Test; @@ -61,7 +62,7 @@ public void testFindOptionsInvalidOffset() { assertEquals(collection.find(skipBy(10).limit(1)).size(), 0); } - @Test(expected = ValidationException.class) + @Test(expected = InvalidOperationException.class) public void testFindInvalidSort() { insert(); collection.find(orderBy("data", SortOrder.Descending)).toList(); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java index f5e5a2c04..7dedc072f 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java @@ -420,7 +420,7 @@ public void testFindWithIterableEqual() { new ArrayList() {{ add("three"); add("four"); - add("three"); + add("five"); }})); assertNotNull(ids); assertEquals(ids.size(), 1); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java index 247a1705a..4a45fa055 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java @@ -188,7 +188,7 @@ public void testRebuildIndex() { insert(); Collection indices = collection.listIndices(); for (IndexDescriptor idx : indices) { - collection.rebuildIndex(idx.getIndexFields().getFieldNames().toArray(new String[0])); + collection.rebuildIndex(idx.getFields().getFieldNames().toArray(new String[0])); } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java index fe37845e1..a59e7e459 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java @@ -134,7 +134,7 @@ public void testRebuildIndex() { insert(); Collection indices = collection.listIndices(); for (IndexDescriptor idx : indices) { - collection.rebuildIndex(idx.getIndexFields().getFieldNames().toArray(new String[0])); + collection.rebuildIndex(idx.getFields().getFieldNames().toArray(new String[0])); } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java index 0055f97ac..08cf8fc9f 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java @@ -88,6 +88,7 @@ public Document processAfterRead(Document document) { .put("expiryDate", new Date()); collection.insert(document); + cvvProcessor.process(collection); collection.addProcessor(cvvProcessor); } @@ -198,15 +199,4 @@ public void testIndexOnEncryptedField() { Document document = collection.find(where("cvv").eq("008")).firstOrNull(); assertNull(document); } - - @Test - public void testRemoveProcessor() { - Document document = collection.find(where("cvv").eq("008")).firstOrNull(); - assertNull(document); - - collection.removeProcessor(cvvProcessor); - - document = collection.find(where("cvv").eq("008")).firstOrNull(); - assertNotNull(document); - } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java index ad3534435..ee6963f91 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/NitriteCollectionTest.java @@ -18,7 +18,7 @@ package org.dizitart.no2.integration.collection; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java index 95baec828..9bbbcbdb1 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/event/EventTest.java @@ -17,14 +17,13 @@ package org.dizitart.no2.integration.event; -import org.dizitart.no2.integration.Retry; -import org.dizitart.no2.integration.repository.data.Employee; import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.events.EventType; -import org.dizitart.no2.repository.ObjectRepository; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; -import org.dizitart.no2.rocksdb.RocksDBModule; import org.dizitart.no2.integration.repository.data.Employee; +import org.dizitart.no2.repository.ObjectRepository; +import org.dizitart.no2.rocksdb.RocksDBModule; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -40,6 +39,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.awaitility.Awaitility.await; +import static org.dizitart.no2.collection.UpdateOptions.updateOptions; import static org.dizitart.no2.filters.Filter.ALL; import static org.dizitart.no2.filters.FluentFilter.where; import static org.dizitart.no2.integration.TestUtil.deleteDb; @@ -88,6 +88,9 @@ public void setUp() { .openOrCreate(); } + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new Employee.EmployeeConverter()); + employeeRepository = db.getRepository(Employee.class); listener = new SampleListenerCollection(); employeeRepository.subscribe(listener); @@ -129,7 +132,7 @@ public void testUpsert() { e.setEmpId(1L); e.setAddress("abcd"); - employeeRepository.update(where("empId").eq(1), e, true); + employeeRepository.update(where("empId").eq(1), e, updateOptions(true)); await().atMost(1, TimeUnit.SECONDS).until(listenerPrepared(EventType.Insert)); assertEquals(listener.getAction(), EventType.Insert); assertNotNull(listener.getItem()); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java index cb6128ba7..cd31138bd 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/MigrationTest.java @@ -23,11 +23,12 @@ import org.dizitart.no2.collection.NitriteCollection; import org.dizitart.no2.common.Constants; import org.dizitart.no2.common.Fields; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.exceptions.MigrationException; import org.dizitart.no2.index.IndexOptions; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.integration.Retry; -import org.dizitart.no2.migration.Instructions; +import org.dizitart.no2.migration.InstructionSet; import org.dizitart.no2.migration.Migration; import org.dizitart.no2.migration.TypeConverter; import org.dizitart.no2.repository.ObjectRepository; @@ -57,6 +58,12 @@ public class MigrationTest { @Before public void setUp() { db = createDb(dbPath); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new OldClass.Converter()); + documentMapper.registerEntityConverter(new OldClass.Literature.Converter()); + documentMapper.registerEntityConverter(new NewClass.Converter()); + documentMapper.registerEntityConverter(new NewClass.Literature.Converter()); + faker = new Faker(); } @@ -91,7 +98,7 @@ public void testRepositoryMigrate() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forDatabase() .addPassword("test-user", "test-password"); @@ -120,10 +127,16 @@ public void migrate(Instructions instruction) { .addMigrations(migration) .openOrCreate("test-user", "test-password"); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new OldClass.Converter()); + documentMapper.registerEntityConverter(new OldClass.Literature.Converter()); + documentMapper.registerEntityConverter(new NewClass.Converter()); + documentMapper.registerEntityConverter(new NewClass.Literature.Converter()); + ObjectRepository newRepo = db.getRepository(NewClass.class); assertEquals(newRepo.size(), 10); assertTrue(db.listCollectionNames().isEmpty()); - assertTrue(db.listKeyedRepository().isEmpty()); + assertTrue(db.listKeyedRepositories().isEmpty()); assertEquals((int) db.getDatabaseMetaData().getSchemaVersion(), 2); } @@ -147,7 +160,7 @@ public void testCollectionMigrate() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forDatabase() .addPassword("test-user", "test-password"); @@ -177,11 +190,11 @@ public void migrate(Instructions instruction) { migration = new Migration(2, 3) { @Override - public void migrate(Instructions instructions) { - instructions.forDatabase() + public void migrate(InstructionSet instructionSet) { + instructionSet.forDatabase() .changePassword("test-user", "test-password", "password"); - instructions.forCollection("testCollectionMigrate") + instructionSet.forCollection("testCollectionMigrate") .dropIndex("firstName") .deleteField("bloodGroup") .addField("name", document -> faker.name().fullName()) @@ -227,7 +240,7 @@ public void testOpenWithoutSchemaVersion() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testOpenWithoutSchemaVersion") @@ -277,7 +290,7 @@ public void testDescendingSchema() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testDescendingSchema") @@ -302,9 +315,9 @@ public void migrate(Instructions instruction) { migration = new Migration(2, Constants.INITIAL_SCHEMA_VERSION) { @Override - public void migrate(Instructions instructions) { + public void migrate(InstructionSet instructionSet) { - instructions.forCollection("testDescendingSchema") + instructionSet.forCollection("testDescendingSchema") .rename("test"); } }; @@ -339,7 +352,7 @@ public void testMigrationWithoutVersion() { Migration migration = new Migration(Constants.INITIAL_SCHEMA_VERSION, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .rename("testMigrationWithoutVersion") @@ -379,7 +392,7 @@ public void testWrongSchemaVersionNoMigration() { Migration migration = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testWrongSchemaVersionNoMigration") .rename("test") @@ -407,8 +420,8 @@ public void migrate(Instructions instruction) { migration = new Migration(2, 3) { @Override - public void migrate(Instructions instructions) { - instructions.forCollection("test") + public void migrate(InstructionSet instructionSet) { + instructionSet.forCollection("test") .rename("testWrongSchemaVersionNoMigration"); } }; @@ -446,7 +459,7 @@ public void testReOpenAfterMigration() { Migration migration = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testReOpenAfterMigration") .rename("test") @@ -513,7 +526,7 @@ public void testMultipleMigrations() { Migration migration1 = new Migration(1, 2) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("testMultipleMigrations") .rename("test"); @@ -522,7 +535,7 @@ public void migrate(Instructions instruction) { Migration migration2 = new Migration(2, 3) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .addField("fullName", "Dummy Name"); } @@ -546,7 +559,7 @@ public void migrate(Instructions instruction) { Migration migration3 = new Migration(3, 4) { @Override - public void migrate(Instructions instruction) { + public void migrate(InstructionSet instruction) { instruction.forCollection("test") .addField("age", 10); } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java index 56ef25c22..49da00aa2 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/NewClass.java @@ -19,8 +19,8 @@ import lombok.Data; import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Entity; import org.dizitart.no2.repository.annotations.Id; @@ -31,11 +31,11 @@ */ @Data @Entity(value = "new", indices = { - @Index(value = "familyName", type = IndexType.NON_UNIQUE), - @Index(value = "fullName", type = IndexType.NON_UNIQUE), - @Index(value = "literature.ratings", type = IndexType.NON_UNIQUE), + @Index(fields = "familyName", type = IndexType.NON_UNIQUE), + @Index(fields = "fullName", type = IndexType.NON_UNIQUE), + @Index(fields = "literature.ratings", type = IndexType.NON_UNIQUE), }) -public class NewClass implements Mappable { +public class NewClass { @Id private Long empId; private String firstName; @@ -43,42 +43,61 @@ public class NewClass implements Mappable { private String fullName; private Literature literature; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("empId", empId) - .put("firstName", firstName) - .put("familyName", familyName) - .put("fullName", fullName) - .put("literature", literature.write(mapper)); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return NewClass.class; + } + + @Override + public Document toDocument(NewClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument("empId", entity.empId) + .put("firstName", entity.firstName) + .put("familyName", entity.familyName) + .put("fullName", entity.fullName) + .put("literature", nitriteMapper.convert(entity.literature, Document.class)); + } - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - firstName = document.get("firstName", String.class); - familyName = document.get("familyName", String.class); - fullName = document.get("fullName", String.class); + @Override + public NewClass fromDocument(Document document, NitriteMapper nitriteMapper) { + NewClass entity = new NewClass(); + entity.empId = document.get("empId", Long.class); + entity.firstName = document.get("firstName", String.class); + entity.familyName = document.get("familyName", String.class); + entity.fullName = document.get("fullName", String.class); - Document doc = document.get("literature", Document.class); - literature = new Literature(); - literature.read(mapper, doc); + Document doc = document.get("literature", Document.class); + entity.literature = nitriteMapper.convert(doc, Literature.class); + return entity; + } } @Data - public static class Literature implements Mappable { + public static class Literature { private String text; private Integer ratings; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("text", text) - .put("ratings", ratings); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - text = document.get("text", String.class); - ratings = document.get("ratings", Integer.class); + @Override + public Class getEntityType() { + return Literature.class; + } + + @Override + public Document toDocument(Literature entity, NitriteMapper nitriteMapper) { + return Document.createDocument("text", entity.text) + .put("ratings", entity.ratings); + } + + @Override + public Literature fromDocument(Document document, NitriteMapper nitriteMapper) { + Literature entity = new Literature(); + entity.text = document.get("text", String.class); + entity.ratings = document.get("ratings", Integer.class); + return entity; + } } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java index 3bbb09b4e..c4395b7ba 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/migrate/OldClass.java @@ -19,8 +19,8 @@ import lombok.Data; import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Entity; import org.dizitart.no2.repository.annotations.Id; @@ -31,12 +31,12 @@ */ @Data @Entity(value = "old", indices = { - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "lastName", type = IndexType.NON_UNIQUE), - @Index(value = "literature.text", type = IndexType.FULL_TEXT), - @Index(value = "literature.ratings", type = IndexType.NON_UNIQUE), + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "lastName", type = IndexType.NON_UNIQUE), + @Index(fields = "literature.text", type = IndexType.FULL_TEXT), + @Index(fields = "literature.ratings", type = IndexType.NON_UNIQUE), }) -public class OldClass implements Mappable { +public class OldClass { @Id private String uuid; private String empId; @@ -44,42 +44,61 @@ public class OldClass implements Mappable { private String lastName; private Literature literature; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("empId", empId) - .put("uuid", uuid) - .put("firstName", firstName) - .put("lastName", lastName) - .put("literature", literature.write(mapper)); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return OldClass.class; + } + + @Override + public Document toDocument(OldClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument("empId", entity.empId) + .put("uuid", entity.uuid) + .put("firstName", entity.firstName) + .put("lastName", entity.lastName) + .put("literature", nitriteMapper.convert(entity.literature, Document.class)); + } - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", String.class); - uuid = document.get("uuid", String.class); - firstName = document.get("firstName", String.class); - lastName = document.get("lastName", String.class); + @Override + public OldClass fromDocument(Document document, NitriteMapper nitriteMapper) { + OldClass entity = new OldClass(); + entity.empId = document.get("empId", String.class); + entity.uuid = document.get("uuid", String.class); + entity.firstName = document.get("firstName", String.class); + entity.lastName = document.get("lastName", String.class); - Document doc = document.get("literature", Document.class); - literature = new Literature(); - literature.read(mapper, doc); + Document doc = document.get("literature", Document.class); + entity.literature = nitriteMapper.convert(doc, Literature.class); + return entity; + } } @Data - public static class Literature implements Mappable { + public static class Literature { private String text; private Float ratings; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("text", text) - .put("ratings", ratings); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - text = document.get("text", String.class); - ratings = document.get("ratings", Float.class); + @Override + public Class getEntityType() { + return Literature.class; + } + + @Override + public Document toDocument(Literature entity, NitriteMapper nitriteMapper) { + return Document.createDocument("text", entity.text) + .put("ratings", entity.ratings); + } + + @Override + public Literature fromDocument(Document document, NitriteMapper nitriteMapper) { + Literature entity = new Literature(); + entity.text = document.get("text", String.class); + entity.ratings = document.get("ratings", Float.class); + return entity; + } } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java index 345d365b2..0f34b6136 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java @@ -19,8 +19,14 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.NitriteBuilder; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.*; +import org.dizitart.no2.integration.repository.decorator.ManufacturerConverter; +import org.dizitart.no2.integration.repository.decorator.MiniProduct; +import org.dizitart.no2.integration.repository.decorator.ProductConverter; +import org.dizitart.no2.integration.repository.decorator.ProductIdConverter; +import org.dizitart.no2.integration.transaction.TxData; import org.dizitart.no2.repository.ObjectRepository; import org.dizitart.no2.rocksdb.RocksDBModule; import org.junit.After; @@ -102,6 +108,33 @@ protected void openDb() { } else { db = nitriteBuilder.openOrCreate(); } + + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new RepositoryJoinTest.Person.Converter()); + documentMapper.registerEntityConverter(new RepositoryJoinTest.Address.Converter()); + documentMapper.registerEntityConverter(new RepositoryJoinTest.PersonDetails.Converter()); + documentMapper.registerEntityConverter(new Company.CompanyConverter()); + documentMapper.registerEntityConverter(new Employee.EmployeeConverter()); + documentMapper.registerEntityConverter(new Note.NoteConverter()); + documentMapper.registerEntityConverter(new Book.BookConverter()); + documentMapper.registerEntityConverter(new BookId.BookIdConverter()); + documentMapper.registerEntityConverter(new ClassA.ClassAConverter()); + documentMapper.registerEntityConverter(new ClassBConverter()); + documentMapper.registerEntityConverter(new ClassC.ClassCConverter()); + documentMapper.registerEntityConverter(new ElemMatch.Converter()); + documentMapper.registerEntityConverter(new InternalClass.Converter()); + documentMapper.registerEntityConverter(new UniversalTextTokenizerTest.TextData.Converter()); + documentMapper.registerEntityConverter(new SubEmployee.Converter()); + documentMapper.registerEntityConverter(new ProductScore.Converter()); + documentMapper.registerEntityConverter(new PersonEntity.Converter()); + documentMapper.registerEntityConverter(new RepeatableIndexTest.Converter()); + documentMapper.registerEntityConverter(new EncryptedPerson.Converter()); + documentMapper.registerEntityConverter(new TxData.Converter()); + documentMapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter()); + documentMapper.registerEntityConverter(new ProductConverter()); + documentMapper.registerEntityConverter(new ProductIdConverter()); + documentMapper.registerEntityConverter(new ManufacturerConverter()); + documentMapper.registerEntityConverter(new MiniProduct.Converter()); } @After diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java index a89c10612..a096fdfb9 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java @@ -24,8 +24,9 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.NitriteConfig; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.Company; @@ -70,6 +71,11 @@ public void setUp() { .fieldSeparator(":") .openOrCreate(); + SimpleDocumentMapper mapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + mapper.registerEntityConverter(new Company.CompanyConverter()); + mapper.registerEntityConverter(new EmployeeForCustomSeparator.EmployeeForCustomSeparatorConverter()); + mapper.registerEntityConverter(new Note.NoteConverter()); + repository = db.getRepository(EmployeeForCustomSeparator.class); } @@ -117,11 +123,11 @@ public void testFindByEmbeddedField() { @ToString @EqualsAndHashCode @Indices({ - @Index(value = "joinDate", type = IndexType.NON_UNIQUE), - @Index(value = "address", type = IndexType.FULL_TEXT), - @Index(value = "employeeNote:text", type = IndexType.FULL_TEXT) + @Index(fields = "joinDate", type = IndexType.NON_UNIQUE), + @Index(fields = "address", type = IndexType.FULL_TEXT), + @Index(fields = "employeeNote:text", type = IndexType.FULL_TEXT) }) - public static class EmployeeForCustomSeparator implements Serializable, Mappable { + public static class EmployeeForCustomSeparator implements Serializable { @Id @Getter @Setter @@ -159,28 +165,39 @@ public EmployeeForCustomSeparator(EmployeeForCustomSeparator copy) { employeeNote = copy.employeeNote; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument().put("empId", empId) - .put("joinDate", joinDate) - .put("address", address) - .put("blob", blob) - .put("company", company.write(mapper)) - .put("employeeNote", employeeNote.write(mapper)); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - joinDate = document.get("joinDate", Date.class); - address = document.get("address", String.class); - blob = document.get("blob", byte[].class); - employeeNote = new Note(); - Document doc = document.get("employeeNote", Document.class); - employeeNote.read(mapper, doc); - company = new Company(); - doc = document.get("company", Document.class); - company.read(mapper, doc); + public static class EmployeeForCustomSeparatorConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return EmployeeForCustomSeparator.class; + } + + @Override + public Document toDocument(EmployeeForCustomSeparator entity, NitriteMapper nitriteMapper) { + return Document.createDocument().put("empId", entity.empId) + .put("joinDate", entity.joinDate) + .put("address", entity.address) + .put("blob", entity.blob) + .put("company", nitriteMapper.convert(entity.company, Document.class)) + .put("employeeNote", nitriteMapper.convert(entity.employeeNote, Document.class)); + } + + @Override + public EmployeeForCustomSeparator fromDocument(Document document, NitriteMapper nitriteMapper) { + EmployeeForCustomSeparator entity = new EmployeeForCustomSeparator(); + + entity.empId = document.get("empId", Long.class); + entity.joinDate = document.get("joinDate", Date.class); + entity.address = document.get("address", String.class); + entity.blob = document.get("blob", byte[].class); + + Document doc = document.get("employeeNote", Document.class); + entity.employeeNote = nitriteMapper.convert(doc, Note.class); + + doc = document.get("company", Document.class); + entity.company = nitriteMapper.convert(doc, Company.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java index 73a093082..8f31d6408 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java @@ -63,6 +63,10 @@ public void setUp() { persons.insert(person); + // process existing data + fieldProcessor.process(persons); + + // add for further changes persons.addProcessor(fieldProcessor); person = new EncryptedPerson(); @@ -187,21 +191,4 @@ public void testIndexOnEncryptedField() { EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull(); assertNull(person); } - - @Test - public void testRemoveProcessor() { - EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull(); - assertNull(person); - - person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull(); - assertNull(person); - - persons.removeProcessor(fieldProcessor); - - person = persons.find(where("cvv").eq("008")).firstOrNull(); - assertNotNull(person); - - person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull(); - assertNotNull(person); - } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java index 1a52708ce..e6a2a58d4 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -27,20 +27,29 @@ * @author Anindya Chatterjee. */ @Data -class InternalClass implements Mappable { +class InternalClass { @Id - private long id; + private Long id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("name", name); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return InternalClass.class; + } + + @Override + public Document toDocument(InternalClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - name = document.get("name", String.class); + @Override + public InternalClass fromDocument(Document document, NitriteMapper nitriteMapper) { + InternalClass entity = new InternalClass(); + entity.id = document.get("id", Long.class); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java index dedd5e256..4b776ea17 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java @@ -17,6 +17,7 @@ package org.dizitart.no2.integration.repository; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.WithNitriteId; import org.dizitart.no2.Nitrite; @@ -53,6 +54,9 @@ public class NitriteIdAsIdTest { @Before public void before() { db = TestUtil.createDb(fileName); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter()); + repo = db.getRepository(WithNitriteId.class); } @@ -90,7 +94,7 @@ public void testNitriteIdField() { } @Test(expected = InvalidIdException.class) - public void setIdDuringInsert() { + public void testSetIdDuringInsert() { WithNitriteId item1 = new WithNitriteId(); item1.name = "first"; item1.idField = NitriteId.newId(); @@ -99,7 +103,7 @@ public void setIdDuringInsert() { } @Test - public void changeIdDuringUpdate() { + public void testChangeIdDuringUpdate() { WithNitriteId item2 = new WithNitriteId(); item2.name = "second"; WriteResult result = repo.insert(item2); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java index e070d485d..6d7135ef5 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java @@ -17,6 +17,7 @@ package org.dizitart.no2.integration.repository; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.*; import org.dizitart.no2.Nitrite; @@ -50,6 +51,18 @@ public class ObjectRepositoryNegativeTest { @Before public void setUp() { db = TestUtil.createDb(dbPath); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new WithPublicField.Converter()); + documentMapper.registerEntityConverter(new WithObjectId.Converter()); + documentMapper.registerEntityConverter(new WithOutId.Converter()); + documentMapper.registerEntityConverter(new WithoutEmbeddedId.Converter()); + documentMapper.registerEntityConverter(new WithoutEmbeddedId.NestedId.Converter()); + documentMapper.registerEntityConverter(new WithEmptyStringId.Converter()); + documentMapper.registerEntityConverter(new WithNullId.Converter()); + documentMapper.registerEntityConverter(new Employee.EmployeeConverter()); + documentMapper.registerEntityConverter(new Company.CompanyConverter()); + documentMapper.registerEntityConverter(new Note.NoteConverter()); + documentMapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter()); } @After @@ -59,7 +72,7 @@ public void close() throws IOException { deleteDb(dbPath); } - @Test(expected = ObjectMappingException.class) + @Test(expected = ValidationException.class) public void testWithCircularReference() { ObjectRepository repository = db.getRepository(WithCircularReference.class); @@ -79,7 +92,7 @@ public void testWithCircularReference() { } } - @Test(expected = ObjectMappingException.class) + @Test(expected = ValidationException.class) public void testWithCustomConstructor() { ObjectRepository repository = db.getRepository(WithCustomConstructor.class); @@ -132,18 +145,6 @@ public void testFindResultRemove() { result.iterator().remove(); } - @Test(expected = IndexingException.class) - public void testWithObjectId() { - ObjectRepository repository = db.getRepository(WithObjectId.class); - WithOutId id = new WithOutId(); - id.setName("test"); - id.setNumber(1); - - WithObjectId object = new WithObjectId(); - object.setWithOutId(id); - repository.insert(object); - } - @Test(expected = NotIdentifiableException.class) public void testUpdateNoId() { ObjectRepository repository = db.getRepository(WithOutId.class); @@ -185,7 +186,7 @@ public void testGetByNullId() { ObjectRepository repository = db.getRepository(WithPublicField.class); WithPublicField object = new WithPublicField(); object.name = "test"; - object.number = 2; + object.number = 2L; repository.insert(object); WithPublicField instance = repository.getById(null); @@ -221,7 +222,7 @@ public void testGetByWrongIdType() { ObjectRepository repository = db.getRepository(WithPublicField.class); WithPublicField object = new WithPublicField(); object.name = "test"; - object.number = 2; + object.number = 2L; repository.insert(object); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java index 6b2cdc55a..360ee228d 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java @@ -22,14 +22,18 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.MappableMapper; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.meta.Attributes; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.exceptions.ValidationException; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.integration.Retry; import org.dizitart.no2.integration.repository.data.*; +import org.dizitart.no2.integration.repository.decorator.ManufacturerConverter; +import org.dizitart.no2.integration.repository.decorator.MiniProduct; +import org.dizitart.no2.integration.repository.decorator.ProductConverter; +import org.dizitart.no2.integration.repository.decorator.ProductIdConverter; import org.dizitart.no2.repository.Cursor; import org.dizitart.no2.repository.ObjectRepository; import org.dizitart.no2.repository.annotations.Entity; @@ -66,7 +70,25 @@ public class ObjectRepositoryTest { @Before public void setUp() { - NitriteMapper mapper = new MappableMapper(); + SimpleDocumentMapper mapper = new SimpleDocumentMapper(); + mapper.registerEntityConverter(new InternalClass.Converter()); + mapper.registerEntityConverter(new EmployeeEntity.Converter()); + mapper.registerEntityConverter(new StressRecord.Converter()); + mapper.registerEntityConverter(new WithClassField.Converter()); + mapper.registerEntityConverter(new WithDateId.Converter()); + mapper.registerEntityConverter(new WithTransientField.Converter()); + mapper.registerEntityConverter(new WithOutId.Converter()); + mapper.registerEntityConverter(new ChildClass.Converter()); + mapper.registerEntityConverter(new WithOutGetterSetter.Converter()); + mapper.registerEntityConverter(new WithPrivateConstructor.Converter()); + mapper.registerEntityConverter(new WithPublicField.Converter()); + mapper.registerEntityConverter(new Employee.EmployeeConverter()); + mapper.registerEntityConverter(new Company.CompanyConverter()); + mapper.registerEntityConverter(new ProductConverter()); + mapper.registerEntityConverter(new ProductIdConverter()); + mapper.registerEntityConverter(new ManufacturerConverter()); + mapper.registerEntityConverter(new MiniProduct.Converter()); + RocksDBModule storeModule = RocksDBModule.withConfig() .filePath(dbPath) .build(); @@ -115,7 +137,7 @@ public void testWithOutId() { ObjectRepository repository = db.getRepository(WithOutId.class); WithOutId object = new WithOutId(); object.setName("test"); - object.setNumber(2); + object.setNumber(2L); repository.insert(object); for (WithOutId instance : repository.find()) { @@ -129,7 +151,7 @@ public void testWithPublicField() { ObjectRepository repository = db.getRepository(WithPublicField.class); WithPublicField object = new WithPublicField(); object.name = "test"; - object.number = 2; + object.number = 2L; repository.insert(object); WithPublicField instance = repository.getById("test"); @@ -141,7 +163,7 @@ public void testWithPublicField() { public void testWithTransientField() { ObjectRepository repository = db.getRepository(WithTransientField.class); WithTransientField object = new WithTransientField(); - object.setNumber(2); + object.setNumber(2L); object.setName("test"); repository.insert(object); @@ -180,7 +202,7 @@ public void testWriteThousandRecords() { public void testWithPackagePrivateClass() { ObjectRepository repository = db.getRepository(InternalClass.class); InternalClass internalClass = new InternalClass(); - internalClass.setId(1); + internalClass.setId(1L); internalClass.setName("name"); repository.insert(internalClass); @@ -293,7 +315,7 @@ public void testKeyedRepository() { assertTrue(db.hasRepository(Employee.class, "developers")); assertEquals(db.listRepositories().size(), 1); - assertEquals(db.listKeyedRepository().size(), 2); + assertEquals(db.listKeyedRepositories().size(), 2); assertEquals(employeeRepo.find(where("address").text("abcd")).size(), 1); assertEquals(employeeRepo.find(where("address").text("xyz")).size(), 1); @@ -322,7 +344,7 @@ public void testEntityRepository() { assertTrue(errored); assertTrue(db.listRepositories().contains("entity.employee")); - assertEquals(db.listKeyedRepository().size(), 2); + assertEquals(db.listKeyedRepositories().size(), 2); assertEquals(db.listCollectionNames().size(), 0); assertTrue(managerRepo.hasIndex("firstName")); @@ -331,7 +353,7 @@ public void testEntityRepository() { assertTrue(employeeRepo.hasIndex("lastName")); managerRepo.drop(); - assertEquals(db.listKeyedRepository().size(), 1); + assertEquals(db.listKeyedRepositories().size(), 1); } @Test @@ -347,10 +369,10 @@ public void testIssue217() { @Data @Entity(value = "entity.employee", indices = { - @Index(value = "firstName", type = IndexType.NON_UNIQUE), - @Index(value = "lastName", type = IndexType.NON_UNIQUE), + @Index(fields = "firstName", type = IndexType.NON_UNIQUE), + @Index(fields = "lastName", type = IndexType.NON_UNIQUE), }) - private static class EmployeeEntity implements Mappable { + private static class EmployeeEntity { private static final Faker faker = new Faker(); @Id @@ -364,18 +386,28 @@ public EmployeeEntity() { lastName = faker.name().lastName(); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("firstName", firstName) - .put("lastName", lastName); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - firstName = document.get("firstName", String.class); - lastName = document.get("lastName", String.class); + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return EmployeeEntity.class; + } + + @Override + public Document toDocument(EmployeeEntity entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("firstName", entity.firstName) + .put("lastName", entity.lastName); + } + + @Override + public EmployeeEntity fromDocument(Document document, NitriteMapper nitriteMapper) { + EmployeeEntity entity = new EmployeeEntity(); + entity.id = document.get("id", Long.class); + entity.firstName = document.get("firstName", String.class); + entity.lastName = document.get("lastName", String.class); + return entity; + } } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java index 3c89260ac..0e8e7d2bb 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java @@ -20,7 +20,7 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.*; import org.dizitart.no2.collection.events.CollectionEventListener; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.common.WriteResult; import org.dizitart.no2.common.concurrent.LockService; import org.dizitart.no2.common.processors.Processor; @@ -66,10 +66,10 @@ public void testRepositoryFactory() { @Test(expected = ValidationException.class) public void testNullType() { RepositoryFactory factory = new RepositoryFactory(new CollectionFactory(new LockService())); - factory.getRepository(db.getConfig(), null, "dummy"); + factory.getRepository(db.getConfig(), (Class) null, "dummy"); } - @Test + @Test(expected = ValidationException.class) public void testNullCollection() { RepositoryFactory factory = new RepositoryFactory(new CollectionFactory(new LockService())); factory.getRepository(db.getConfig(), DummyCollection.class, null); @@ -136,11 +136,6 @@ public void addProcessor(Processor processor) { } - @Override - public void removeProcessor(Processor processor) { - - } - @Override public void createIndex(IndexOptions indexOptions, String... fields) { diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java index 4dacdbff8..05634377f 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java @@ -22,7 +22,7 @@ import org.dizitart.no2.collection.NitriteId; import org.dizitart.no2.common.Lookup; import org.dizitart.no2.common.RecordStream; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.repository.ObjectRepository; @@ -31,10 +31,7 @@ import org.junit.Before; import org.junit.Test; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; +import java.util.*; import static org.dizitart.no2.collection.Document.createDocument; import static org.dizitart.no2.collection.FindOptions.skipBy; @@ -138,80 +135,120 @@ public void testRemove() { } @Data - public static class Person implements Mappable { + public static class Person { @Id private NitriteId nitriteId; private String id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return createDocument() - .put("nitriteId", nitriteId) - .put("id", id) - .put("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return Person.class; + } + + @Override + public Document toDocument(Person entity, NitriteMapper nitriteMapper) { + return createDocument() + .put("nitriteId", entity.nitriteId) + .put("id", entity.id) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - nitriteId = document.get("nitriteId", NitriteId.class); - id = document.get("id", String.class); - name = document.get("name", String.class); + @Override + public Person fromDocument(Document document, NitriteMapper nitriteMapper) { + Person entity = new Person(); + entity.nitriteId = document.get("nitriteId", NitriteId.class); + entity.id = document.get("id", String.class); + entity.name = document.get("name", String.class); + return entity; + } } } @Data - public static class Address implements Mappable { + public static class Address { @Id private NitriteId nitriteId; private String personId; private String street; - @Override - public Document write(NitriteMapper mapper) { - return createDocument() - .put("nitriteId", nitriteId) - .put("personId", personId) - .put("street", street); - } + public static class Converter implements EntityConverter
{ + + @Override + public Class
getEntityType() { + return Address.class; + } + + @Override + public Document toDocument(Address entity, NitriteMapper nitriteMapper) { + return createDocument() + .put("nitriteId", entity.nitriteId) + .put("personId", entity.personId) + .put("street", entity.street); + } - @Override - public void read(NitriteMapper mapper, Document document) { - nitriteId = document.get("nitriteId", NitriteId.class); - personId = document.get("personId", String.class); - street = document.get("street", String.class); + @Override + public Address fromDocument(Document document, NitriteMapper nitriteMapper) { + Address entity = new Address(); + entity.nitriteId = document.get("nitriteId", NitriteId.class); + entity.personId = document.get("personId", String.class); + entity.street = document.get("street", String.class); + return entity; + } } } @Data - public static class PersonDetails implements Mappable { + public static class PersonDetails { @Id private NitriteId nitriteId; private String id; private String name; private List
addresses; - @Override - public Document write(NitriteMapper mapper) { - return createDocument() - .put("nitriteId", nitriteId) - .put("personId", id) - .put("street", name) - .put("addresses", addresses); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return PersonDetails.class; + } + + @Override + public Document toDocument(PersonDetails entity, NitriteMapper nitriteMapper) { + List documents = new ArrayList<>(); + if (entity.addresses != null) { + for (Address address : entity.addresses) { + documents.add(nitriteMapper.convert(address, Document.class)); + } + } + + return createDocument() + .put("nitriteId", entity.nitriteId) + .put("personId", entity.id) + .put("street", entity.name) + .put("addresses", documents); + } + + @Override + public PersonDetails fromDocument(Document document, NitriteMapper nitriteMapper) { + PersonDetails entity = new PersonDetails(); + + entity.nitriteId = document.get("nitriteId", NitriteId.class); + entity.id = document.get("id", String.class); + entity.name = document.get("name", String.class); + + Collection documents = document.get("addresses", Collection.class); + if (documents != null) { + entity.addresses = new ArrayList<>(); + for (Document doc : documents) { + Address address = nitriteMapper.convert(doc, Address.class); + entity.addresses.add(address); + } + } - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - nitriteId = document.get("nitriteId", NitriteId.class); - id = document.get("id", String.class); - name = document.get("name", String.class); - Set documents = document.get("addresses", Set.class); - this.addresses = new ArrayList<>(); - for (Document doc : documents) { - Address address = new Address(); - address.read(mapper, doc); - addresses.add(address); + return entity; } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java index d5bd0f5cb..f12c86fdf 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java @@ -39,6 +39,7 @@ import static org.awaitility.Awaitility.await; import static org.dizitart.no2.collection.Document.createDocument; +import static org.dizitart.no2.collection.UpdateOptions.updateOptions; import static org.dizitart.no2.filters.FluentFilter.where; import static org.junit.Assert.*; @@ -210,7 +211,7 @@ public void testUpsertTrue() { employee.setEmployeeNote(empNote1); WriteResult writeResult - = employeeRepository.update(where("empId").eq(12), employee, true); + = employeeRepository.update(where("empId").eq(12), employee, updateOptions(true)); assertEquals(writeResult.getAffectedCount(), 1); result = employeeRepository.find(where("joinDate").eq(joiningDate)); @@ -235,7 +236,7 @@ public void testUpsertFalse() { employee.setEmployeeNote(empNote1); WriteResult writeResult - = employeeRepository.update(where("empId").eq(12), employee, false); + = employeeRepository.update(where("empId").eq(12), employee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 0); result = employeeRepository.find(where("joinDate").eq(joiningDate)); @@ -327,7 +328,7 @@ public void testMultiUpdateWithObject() { update.setAddress("new address"); WriteResult writeResult - = employeeRepository.update(where("joinDate").eq(now), update, false); + = employeeRepository.update(where("joinDate").eq(now), update, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 0); } @@ -361,7 +362,7 @@ public void testUpdateWithChangedId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); assertEquals(count, employeeRepository.size()); @@ -380,7 +381,7 @@ public void testUpdateWithNullId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); } @@ -396,7 +397,7 @@ public void testUpdateWithDuplicateId() { Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull(); assertNotNull(result.getJoinDate()); - WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false); + WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false)); assertEquals(writeResult.getAffectedCount(), 1); assertEquals(count, employeeRepository.size()); @@ -563,12 +564,12 @@ public void testDelete() { public void testUpdateObjectNotExistsUpsertTrue() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(2); + a.setId(2L); a.setName("second"); // it will insert as new object @@ -580,18 +581,18 @@ public void testUpdateObjectNotExistsUpsertTrue() { public void testUpdateObjectNotExistsUpsertFalse() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(2); + a.setId(2L); a.setName("second"); // no changes will happen to repository repo.update(a, false); assertEquals(repo.size(), 1); - assertEquals(repo.find().firstOrNull().getId(), 1); + assertEquals(repo.find().firstOrNull().getId().longValue(), 1); assertEquals(repo.find().firstOrNull().getName(), "first"); } @@ -599,18 +600,18 @@ public void testUpdateObjectNotExistsUpsertFalse() { public void testUpdateObjectExistsUpsertTrue() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("second"); // update existing object, keep id same repo.update(a, true); assertEquals(repo.size(), 1); - assertEquals(repo.find().firstOrNull().getId(), 1); + assertEquals(repo.find().firstOrNull().getId().longValue(), 1); assertEquals(repo.find().firstOrNull().getName(), "second"); } @@ -618,18 +619,18 @@ public void testUpdateObjectExistsUpsertTrue() { public void testUpdateObjectExistsUpsertFalse() { ObjectRepository repo = db.getRepository(InternalClass.class); InternalClass a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("first"); repo.insert(a); a = new InternalClass(); - a.setId(1); + a.setId(1L); a.setName("second"); // update existing object, keep id same repo.update(a, false); assertEquals(repo.size(), 1); - assertEquals(repo.find().firstOrNull().getId(), 1); + assertEquals(repo.find().firstOrNull().getId().longValue(), 1); assertEquals(repo.find().firstOrNull().getName(), "second"); } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java index 68015c9f3..61efe315b 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java @@ -18,15 +18,16 @@ package org.dizitart.no2.integration.repository; import lombok.Getter; -import org.dizitart.no2.integration.repository.data.*; import org.dizitart.no2.collection.Document; import org.dizitart.no2.common.SortOrder; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.exceptions.FilterException; import org.dizitart.no2.exceptions.InvalidIdException; import org.dizitart.no2.exceptions.NotIdentifiableException; import org.dizitart.no2.filters.Filter; +import org.dizitart.no2.integration.repository.data.*; import org.dizitart.no2.repository.Cursor; import org.dizitart.no2.repository.ObjectRepository; import org.junit.Test; @@ -364,21 +365,20 @@ public void testElemMatchFilter() { final ProductScore score6 = new ProductScore("xyz", 8); ObjectRepository repository = db.getRepository(ElemMatch.class); - ElemMatch e1 = new ElemMatch() {{ - setId(1); - setStrArray(new String[]{"a", "b"}); - setProductScores(new ProductScore[]{score1, score4}); - }}; - ElemMatch e2 = new ElemMatch() {{ - setId(2); - setStrArray(new String[]{"d", "e"}); - setProductScores(new ProductScore[]{score2, score5}); - }}; - ElemMatch e3 = new ElemMatch() {{ - setId(3); - setStrArray(new String[]{"a", "f"}); - setProductScores(new ProductScore[]{score3, score6}); - }}; + ElemMatch e1 = new ElemMatch(); + e1.setId(1L); + e1.setStrArray(new String[]{"a", "b"}); + e1.setProductScores(new ProductScore[]{score1, score4}); + + ElemMatch e2 = new ElemMatch(); + e2.setId(2L); + e2.setStrArray(new String[]{"d", "e"}); + e2.setProductScores(new ProductScore[]{score2, score5}); + + ElemMatch e3 = new ElemMatch(); + e3.setId(3L); + e3.setStrArray(new String[]{"a", "f"}); + e3.setProductScores(new ProductScore[]{score3, score6}); repository.insert(e1, e2, e3); @@ -563,24 +563,37 @@ public void testIdSet() { @Test public void testBetweenFilter() { @Getter - class TestData implements Mappable { + class TestData { private Date age; public TestData(Date age) { this.age = age; } + } + + class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return TestData.class; + } @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("age", age); + public Document toDocument(TestData entity, NitriteMapper nitriteMapper) { + return Document.createDocument("age", entity.age); } @Override - public void read(NitriteMapper mapper, Document document) { - age = document.get("age", Date.class); + public TestData fromDocument(Document document, NitriteMapper nitriteMapper) { + TestData entity = new TestData(new Date()); + entity.age = document.get("age", Date.class); + return entity; } } + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new Converter()); + TestData data1 = new TestData(new GregorianCalendar(2020, Calendar.JANUARY, 11).getTime()); TestData data2 = new TestData(new GregorianCalendar(2021, Calendar.FEBRUARY, 12).getTime()); TestData data3 = new TestData(new GregorianCalendar(2022, Calendar.MARCH, 13).getTime()); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java index 8b2237f0b..d509adc70 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java @@ -20,8 +20,9 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.NitriteBuilder; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.index.NitriteTextIndexer; import org.dizitart.no2.index.fulltext.Languages; @@ -54,6 +55,9 @@ public class UniversalTextTokenizerTest extends BaseObjectRepositoryTest { @Override public void setUp() { openDb(); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TextData.Converter()); + textRepository = db.getRepository(TextData.class); for (int i = 0; i < 10; i++) { @@ -166,22 +170,31 @@ public void testUniversalFullTextIndexing() { } @Indices( - @Index(value = "text", type = IndexType.FULL_TEXT) + @Index(fields = "text", type = IndexType.FULL_TEXT) ) - public static class TextData implements Mappable { - public int id; + public static class TextData { + public Integer id; public String text; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("text", text); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return TextData.class; + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Integer.class); - text = document.get("text", String.class); + @Override + public Document toDocument(TextData entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("text", entity.text); + } + + @Override + public TextData fromDocument(Document document, NitriteMapper nitriteMapper) { + TextData entity = new TextData(); + entity.id = document.get("id", Integer.class); + entity.text = document.get("text", String.class); + return entity; + } } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java index 5a147769c..c27993fc4 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Book.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Entity; @@ -35,12 +35,12 @@ */ @Data @Entity(value = "books", indices = { - @Index(value = "tags", type = IndexType.NON_UNIQUE), - @Index(value = "description", type = IndexType.FULL_TEXT), - @Index(value = { "price", "publisher" }) + @Index(fields = "tags", type = IndexType.NON_UNIQUE), + @Index(fields = "description", type = IndexType.FULL_TEXT), + @Index(fields = { "price", "publisher" }) }) -public class Book implements Mappable { - @Id(fieldName = "book_id") +public class Book { + @Id(fieldName = "book_id", embeddedFields = { "isbn", "book_name" }) private BookId bookId; private String publisher; @@ -51,22 +51,32 @@ public class Book implements Mappable { private String description; - @Override - public Document write(NitriteMapper mapper) { - return createDocument("book_id", mapper.convert(bookId, Document.class)) - .put("publisher", publisher) - .put("price", price) - .put("tags", tags) - .put("description", description); - } + public static class BookConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Book.class; + } + + @Override + public Document toDocument(Book entity, NitriteMapper nitriteMapper) { + return createDocument("book_id", nitriteMapper.convert(entity.bookId, Document.class)) + .put("publisher", entity.publisher) + .put("price", entity.price) + .put("tags", entity.tags) + .put("description", entity.description); + } - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - bookId = mapper.convert(document.get("book_id"), BookId.class); - publisher = document.get("publisher", String.class); - price = document.get("price", Double.class); - tags = (List) document.get("tags", List.class); - description = document.get("description", String.class); + @Override + @SuppressWarnings("unchecked") + public Book fromDocument(Document document, NitriteMapper nitriteMapper) { + Book entity = new Book(); + entity.bookId = nitriteMapper.convert(document.get("book_id"), BookId.class); + entity.publisher = document.get("publisher", String.class); + entity.price = document.get("price", Double.class); + entity.tags = (List) document.get("tags", List.class); + entity.description = document.get("description", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java index 239b493d3..4aa11f1af 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java @@ -19,9 +19,8 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; -import org.dizitart.no2.repository.annotations.Embedded; import static org.dizitart.no2.collection.Document.createDocument; @@ -29,26 +28,34 @@ * @author Anindya Chatterjee */ @Data -public class BookId implements Mappable { - @Embedded(order = 0) +public class BookId { private String isbn; - @Embedded(order = 1, fieldName = "book_name") private String name; private String author; - @Override - public Document write(NitriteMapper mapper) { - return createDocument("isbn", isbn) - .put("book_name", name) - .put("author", author); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - isbn = document.get("isbn", String.class); - name = document.get("book_name", String.class); - author = document.get("author", String.class); + public static class BookIdConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return BookId.class; + } + + @Override + public Document toDocument(BookId entity, NitriteMapper nitriteMapper) { + return createDocument("isbn", entity.isbn) + .put("book_name", entity.name) + .put("author", entity.author); + } + + @Override + public BookId fromDocument(Document document, NitriteMapper nitriteMapper) { + BookId entity = new BookId(); + entity.isbn = document.get("isbn", String.class); + entity.name = document.get("book_name", String.class); + entity.author = document.get("author", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java index d3c2ba644..6f0470d26 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java @@ -20,9 +20,12 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.InheritIndices; +import java.util.Date; + /** * @author Anindya Chatterjee */ @@ -32,14 +35,30 @@ public class ChildClass extends ParentClass { private String name; - @Override - public Document write(NitriteMapper mapper) { - return super.write(mapper).put("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return ChildClass.class; + } + + @Override + public Document toDocument(ChildClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("name", entity.getName()) + .put("id", entity.getId()) + .put("date", entity.getDate()) + .put("text", entity.getText()); + } - @Override - public void read(NitriteMapper mapper, Document document) { - super.read(mapper, document); - name = document.get("name", String.class); + @Override + public ChildClass fromDocument(Document document, NitriteMapper nitriteMapper) { + ChildClass entity = new ChildClass(); + entity.setId(document.get("id", Long.class)); + entity.setDate(document.get("date", Date.class)); + entity.setText(document.get("text", String.class)); + entity.setName(document.get("name", String.class)); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java index a8161c4e2..4e736981d 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java @@ -22,14 +22,14 @@ import lombok.Setter; import lombok.ToString; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.UUID; @EqualsAndHashCode @ToString -public class ClassA implements Mappable { +public class ClassA { @Getter @Setter private ClassB b; @@ -53,23 +53,33 @@ public static ClassA create(int seed) { return classA; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("b", b != null ? b.write(mapper) : null) - .put("uid", uid) - .put("string", string) - .put("blob", blob); - } + public static class ClassAConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return ClassA.class; + } + + @Override + public Document toDocument(ClassA entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("b", nitriteMapper.convert(entity.b, Document.class)) + .put("uid", entity.uid) + .put("string", entity.string) + .put("blob", entity.blob); + } - @Override - public void read(NitriteMapper mapper, Document document) { - if (document.get("b") != null) { - b = new ClassB(); - b.read(mapper, document.get("b", Document.class)); + @Override + public ClassA fromDocument(Document document, NitriteMapper nitriteMapper) { + ClassA entity = new ClassA(); + if (document.get("b") != null) { + Document doc = document.get("b", Document.class); + entity.b = nitriteMapper.convert(doc, ClassB.class); + } + entity.uid = document.get("uid", UUID.class); + entity.string = document.get("string", String.class); + entity.blob = document.get("blob", byte[].class); + return entity; } - uid = document.get("uid", UUID.class); - string = document.get("string", String.class); - blob = document.get("blob", byte[].class); } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java index a546b91e2..51cc9b416 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java @@ -21,13 +21,10 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; @EqualsAndHashCode @ToString -class ClassB implements Comparable, Mappable { +class ClassB implements Comparable { @Getter @Setter private int number; @@ -47,16 +44,4 @@ public int compareTo(ClassB o) { return Integer.compare(number, o.number); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("number", number) - .put("text", text); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - number = document.get("number", Integer.class); - text = document.get("text", String.class); - } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java new file mode 100644 index 000000000..6d5f88bab --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.data; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ClassBConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return ClassB.class; + } + + @Override + public Document toDocument(ClassB entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("number", entity.getNumber()) + .put("text", entity.getText()); + } + + @Override + public ClassB fromDocument(Document document, NitriteMapper nitriteMapper) { + ClassB entity = new ClassB(); + if (document.get("number") != null) { + entity.setNumber(document.get("number", Integer.class)); + } + entity.setText(document.get("text", String.class)); + return entity; + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java index 860fe1e93..0dd8d58d6 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java @@ -22,12 +22,12 @@ import lombok.Setter; import lombok.ToString; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; @EqualsAndHashCode @ToString -public class ClassC implements Mappable { +public class ClassC { @Getter @Setter private long id; @@ -46,21 +46,37 @@ public static ClassC create(int seed) { return classC; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("id", id) - .put("digit", digit) - .put("parent", parent != null ? parent.write(mapper) : null); - } + public static class ClassCConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return ClassC.class; + } + + @Override + public Document toDocument(ClassC entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("id", entity.id) + .put("digit", entity.digit) + .put("parent", nitriteMapper.convert(entity.parent, Document.class)); + } + + @Override + public ClassC fromDocument(Document document, NitriteMapper nitriteMapper) { + ClassC entity = new ClassC(); + if (document.get("id") != null) { + entity.id = document.get("id", Long.class); + } + + if (document.get("digit") != null) { + entity.digit = document.get("digit", Double.class); + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - digit = document.get("digit", Double.class); - if (document.get("parent") != null) { - parent = new ClassA(); - parent.read(mapper, document.get("parent", Document.class)); + if (document.get("parent") != null) { + Document doc = document.get("parent", Document.class); + entity.parent = nitriteMapper.convert(doc, ClassA.class); + } + return entity; } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java index 944e17358..649053261 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Company.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -37,9 +37,9 @@ */ @EqualsAndHashCode @Indices({ - @Index(value = "companyName") + @Index(fields = "companyName") }) -public class Company implements Serializable, Mappable { +public class Company implements Serializable { @Id(fieldName = "company_id") @Getter @Setter @@ -61,25 +61,6 @@ public class Company implements Serializable, Mappable { @Setter private Map> employeeRecord; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("company_id", companyId) - .put("companyName", companyName) - .put("dateCreated", dateCreated) - .put("departments", departments) - .put("employeeRecord", employeeRecord); - } - - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - companyId = document.get("company_id", Long.class); - companyName = document.get("companyName", String.class); - dateCreated = document.get("dateCreated", Date.class); - departments = document.get("departments", List.class); - employeeRecord = document.get("employeeRecord", Map.class); - } - @Override public String toString() { return "Company{" + @@ -89,4 +70,32 @@ public String toString() { ", departments=" + departments + '}'; } + + public static class CompanyConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Company.class; + } + + @Override + public Document toDocument(Company entity, NitriteMapper nitriteMapper) { + return Document.createDocument("company_id", entity.companyId) + .put("companyName", entity.companyName) + .put("dateCreated", entity.dateCreated) + .put("departments", entity.departments) + .put("employeeRecord", entity.employeeRecord); + } + + @Override + public Company fromDocument(Document document, NitriteMapper nitriteMapper) { + Company entity = new Company(); + entity.companyId = document.get("company_id", Long.class); + entity.companyName = document.get("companyName", String.class); + entity.dateCreated = document.get("dateCreated", Date.class); + entity.departments = document.get("departments", List.class); + entity.employeeRecord = document.get("employeeRecord", Map.class); + return entity; + } + } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java index 26e0dcf4a..6d83c2940 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java @@ -19,6 +19,9 @@ import com.github.javafaker.Faker; import lombok.val; +import org.dizitart.no2.integration.repository.decorator.Manufacturer; +import org.dizitart.no2.integration.repository.decorator.Product; +import org.dizitart.no2.integration.repository.decorator.ProductId; import java.nio.charset.StandardCharsets; import java.util.*; @@ -102,6 +105,15 @@ public static Book randomBook() { return book; } + public static Product randomProduct() { + Product product = new Product(); + product.setProductName(faker.name().name()); + product.setProductId(randomProductId()); + product.setManufacturer(randomManufacturer()); + product.setPrice(Double.parseDouble(faker.commerce().price())); + return product; + } + private static List departments() { return new ArrayList() {{ add("dev"); @@ -114,4 +126,19 @@ private static List departments() { add("support"); }}; } + + private static ProductId randomProductId() { + ProductId productId = new ProductId(); + productId.setProductCode(faker.code().ean13()); + productId.setUniqueId(UUID.randomUUID().toString()); + return productId; + } + + private static Manufacturer randomManufacturer() { + Manufacturer manufacturer = new Manufacturer(); + manufacturer.setUniqueId(random.nextInt()); + manufacturer.setName(faker.name().name()); + manufacturer.setAddress(faker.address().fullAddress()); + return manufacturer; + } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java index cb5304e29..92c710d2e 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.ArrayList; @@ -29,38 +29,46 @@ * @author Anindya Chatterjee */ @Data -public class ElemMatch implements Mappable { - private long id; +public class ElemMatch { + private Long id; private String[] strArray; private ProductScore[] productScores; - @Override - public Document write(NitriteMapper mapper) { - List list = new ArrayList<>(); - if (productScores != null) { - for (ProductScore productScore : productScores) { - Document document = productScore.write(mapper); - list.add(document); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return ElemMatch.class; } - return Document.createDocument("id", id) - .put("strArray", strArray) - .put("productScores", list); - } + @Override + public Document toDocument(ElemMatch entity, NitriteMapper nitriteMapper) { + List list = new ArrayList<>(); + if (entity.productScores != null) { + for (ProductScore productScore : entity.productScores) { + Document document = nitriteMapper.convert(productScore, Document.class); + list.add(document); + } + } + + return Document.createDocument("id", entity.id) + .put("strArray", entity.strArray) + .put("productScores", list); + } - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - strArray = document.get("strArray", String[].class); - List list = document.get("productScores", List.class); - if (list != null) { - productScores = new ProductScore[list.size()]; - for (int i = 0; i < list.size(); i++) { - productScores[i] = new ProductScore(); - productScores[i].read(mapper, list.get(i)); + @Override + public ElemMatch fromDocument(Document document, NitriteMapper nitriteMapper) { + ElemMatch entity = new ElemMatch(); + entity.id = document.get("id", Long.class); + entity.strArray = document.get("strArray", String[].class); + List list = document.get("productScores", List.class); + if (list != null) { + entity.productScores = new ProductScore[list.size()]; + for (int i = 0; i < list.size(); i++) { + entity.productScores[i] = nitriteMapper.convert(list.get(i), ProductScore.class); + } } + return entity; } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java index 7204f5654..f89b6ecc9 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java @@ -22,9 +22,9 @@ import lombok.Setter; import lombok.ToString; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -36,10 +36,10 @@ */ @ToString @EqualsAndHashCode -@Index(value = "joinDate", type = IndexType.NON_UNIQUE) -@Index(value = "address", type = IndexType.FULL_TEXT) -@Index(value = "employeeNote.text", type = IndexType.FULL_TEXT) -public class Employee implements Serializable, Mappable { +@Index(fields = "joinDate", type = IndexType.NON_UNIQUE) +@Index(fields = "address", type = IndexType.FULL_TEXT) +@Index(fields = "employeeNote.text", type = IndexType.FULL_TEXT) +public class Employee implements Serializable { @Id @Getter @Setter @@ -82,28 +82,39 @@ public Employee(Employee copy) { emailAddress = copy.emailAddress; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("empId", empId) - .put("joinDate", joinDate) - .put("address", address) - .put("blob", blob) - .put("emailAddress", emailAddress) - .put("employeeNote", employeeNote != null ? employeeNote.write(mapper) : null); - } + public static class EmployeeConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Employee.class; + } + + @Override + public Document toDocument(Employee entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("empId", entity.empId) + .put("joinDate", entity.joinDate) + .put("address", entity.address) + .put("blob", entity.blob) + .put("emailAddress", entity.emailAddress) + .put("employeeNote", nitriteMapper.convert(entity.employeeNote, Document.class)); + } + + @Override + public Employee fromDocument(Document document, NitriteMapper nitriteMapper) { + Employee entity = new Employee(); + + entity.empId = document.get("empId", Long.class); + entity.joinDate = document.get("joinDate", Date.class); + entity.address = document.get("address", String.class); + entity.blob = document.get("blob", byte[].class); + entity.emailAddress = document.get("emailAddress", String.class); - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - joinDate = document.get("joinDate", Date.class); - address = document.get("address", String.class); - blob = document.get("blob", byte[].class); - emailAddress = document.get("emailAddress", String.class); - - if (document.get("employeeNote") != null) { - employeeNote = new Note(); - employeeNote.read(mapper, document.get("employeeNote", Document.class)); + if (document.get("employeeNote") != null) { + Document doc = document.get("employeeNote", Document.class); + entity.employeeNote = nitriteMapper.convert(doc, Note.class);; + } + return entity; } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java new file mode 100644 index 000000000..9bf6bcbf6 --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.data; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class EmptyClass { + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return EmptyClass.class; + } + + @Override + public Document toDocument(EmptyClass entity, NitriteMapper nitriteMapper) { + return Document.createDocument(); + } + + @Override + public EmptyClass fromDocument(Document document, NitriteMapper nitriteMapper) { + return new EmptyClass(); + } + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java index 8b9d4b23b..7003db015 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Entity; @@ -30,25 +30,35 @@ */ @Data @Entity -public class EncryptedPerson implements Mappable { +public class EncryptedPerson { private String name; private String creditCardNumber; private String cvv; private Date expiryDate; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("creditCardNumber", creditCardNumber) - .put("cvv", cvv) - .put("expiryDate", expiryDate); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return EncryptedPerson.class; + } + + @Override + public Document toDocument(EncryptedPerson entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("creditCardNumber", entity.creditCardNumber) + .put("cvv", entity.cvv) + .put("expiryDate", entity.expiryDate); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - creditCardNumber = document.get("creditCardNumber", String.class); - cvv = document.get("cvv", String.class); - expiryDate = document.get("expiryDate", Date.class); + @Override + public EncryptedPerson fromDocument(Document document, NitriteMapper nitriteMapper) { + EncryptedPerson entity = new EncryptedPerson(); + entity.name = document.get("name", String.class); + entity.creditCardNumber = document.get("creditCardNumber", String.class); + entity.cvv = document.get("cvv", String.class); + entity.expiryDate = document.get("expiryDate", Date.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java index 67dc307d1..7bf7f1726 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/Note.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.io.Serializable; @@ -30,7 +30,7 @@ * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class Note implements Serializable, Mappable { +public class Note implements Serializable { @Getter @Setter private Long noteId; @@ -38,14 +38,26 @@ public class Note implements Serializable, Mappable { @Setter private String text; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument().put("noteId", noteId).put("text", text); - } + public static class NoteConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Note.class; + } + + @Override + public Document toDocument(Note entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("noteId", entity.noteId) + .put("text", entity.text); + } - @Override - public void read(NitriteMapper mapper, Document document) { - noteId = document.get("noteId", Long.class); - text = document.get("text", String.class); + @Override + public Note fromDocument(Document document, NitriteMapper nitriteMapper) { + Note entity = new Note(); + entity.noteId = document.get("noteId", Long.class); + entity.text = document.get("text", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java index bc2514543..f27a3f869 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java @@ -19,8 +19,6 @@ import lombok.Getter; import lombok.Setter; -import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -31,23 +29,9 @@ */ @Getter @Setter -@Index(value = "date") +@Index(fields = "date") public class ParentClass extends SuperDuperClass { @Id protected Long id; private Date date; - - @Override - public Document write(NitriteMapper mapper) { - return super.write(mapper) - .put("id", id) - .put("date", date); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - super.read(mapper, document); - id = document.get("id", Long.class); - date = document.get("date", Date.class); - } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java index 2b90075fa..b6dcd03a3 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java @@ -19,9 +19,9 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Entity; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; @@ -34,10 +34,10 @@ */ @Data @Entity(value = "MyPerson", indices = { - @Index(value = "name", type = IndexType.FULL_TEXT), - @Index(value = "status", type = IndexType.NON_UNIQUE) + @Index(fields = "name", type = IndexType.FULL_TEXT), + @Index(fields = "status", type = IndexType.NON_UNIQUE) }) -public class PersonEntity implements Mappable { +public class PersonEntity { @Id private String uuid; private String name; @@ -56,24 +56,34 @@ public PersonEntity(String name) { this.dateCreated = new Date(); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("uuid", uuid) - .put("name", name) - .put("status", status) - .put("friend", friend != null ? friend.write(mapper) : null) - .put("dateCreated", dateCreated); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return PersonEntity.class; + } + + @Override + public Document toDocument(PersonEntity entity, NitriteMapper nitriteMapper) { + return Document.createDocument("uuid", entity.uuid) + .put("name", entity.name) + .put("status", entity.status) + .put("friend", entity.friend != null ? nitriteMapper.convert(entity.friend, Document.class) : null) + .put("dateCreated", entity.dateCreated); + } - @Override - public void read(NitriteMapper mapper, Document document) { - if (document != null) { - uuid = document.get("uuid", String.class); - name = document.get("name", String.class); - status = document.get("status", String.class); - dateCreated = document.get("dateCreated", Date.class); - friend = new PersonEntity(); - friend.read(mapper, document.get("friend", Document.class)); + @Override + public PersonEntity fromDocument(Document document, NitriteMapper nitriteMapper) { + if (document != null) { + PersonEntity entity = new PersonEntity(); + entity.uuid = document.get("uuid", String.class); + entity.name = document.get("name", String.class); + entity.status = document.get("status", String.class); + entity.dateCreated = document.get("dateCreated", Date.class); + entity.friend = nitriteMapper.convert(document.get("friend", Document.class), PersonEntity.class); + return entity; + } + return null; } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java index 65550d182..23c9d16ff 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** @@ -28,9 +28,9 @@ */ @Getter @Setter -public class ProductScore implements Mappable { +public class ProductScore { private String product; - private int score; + private Integer score; public ProductScore() { } @@ -40,15 +40,25 @@ public ProductScore(String product, int score) { this.score = score; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("product", product) - .put("score", score); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return ProductScore.class; + } + + @Override + public Document toDocument(ProductScore entity, NitriteMapper nitriteMapper) { + return Document.createDocument("product", entity.product) + .put("score", entity.score); + } - @Override - public void read(NitriteMapper mapper, Document document) { - product = document.get("product", String.class); - score = document.get("score", Integer.class); + @Override + public ProductScore fromDocument(Document document, NitriteMapper nitriteMapper) { + ProductScore entity = new ProductScore(); + entity.product = document.get("product", String.class); + entity.score = document.get("score", Integer.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java index 1ecadb243..5d9208b86 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java @@ -19,34 +19,44 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Index; /** * @author Anindya Chatterjee */ @Data -@Index(value = "firstName") -@Index(value = "age", type = IndexType.NON_UNIQUE) -@Index(value = "lastName", type = IndexType.FULL_TEXT) -public class RepeatableIndexTest implements Mappable { +@Index(fields = "firstName") +@Index(fields = "age", type = IndexType.NON_UNIQUE) +@Index(fields = "lastName", type = IndexType.FULL_TEXT) +public class RepeatableIndexTest { private String firstName; private Integer age; private String lastName; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("firstName", firstName) - .put("age", age) - .put("lastName", lastName); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return RepeatableIndexTest.class; + } + + @Override + public Document toDocument(RepeatableIndexTest entity, NitriteMapper nitriteMapper) { + return Document.createDocument("firstName", entity.firstName) + .put("age", entity.age) + .put("lastName", entity.lastName); + } - @Override - public void read(NitriteMapper mapper, Document document) { - firstName = document.get("firstName", String.class); - age = document.get("age", Integer.class); - lastName = document.get("lastName", String.class); + @Override + public RepeatableIndexTest fromDocument(Document document, NitriteMapper nitriteMapper) { + RepeatableIndexTest entity = new RepeatableIndexTest(); + entity.firstName = document.get("firstName", String.class); + entity.age = document.get("age", Integer.class); + entity.lastName = document.get("lastName", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java index d1cb199c4..33dc06073 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** @@ -28,28 +28,37 @@ */ @Getter @Setter -public class StressRecord implements Mappable { +public class StressRecord { private String firstName; - private boolean processed; + private Boolean processed; private String lastName; - private boolean failed; + private Boolean failed; private String notes; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument().put("firstName", firstName) - .put("processed", processed) - .put("lastName", lastName) - .put("failed", failed) - .put("notes", notes); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return StressRecord.class; + } + + @Override + public Document toDocument(StressRecord entity, NitriteMapper nitriteMapper) { + return Document.createDocument().put("firstName", entity.firstName) + .put("processed", entity.processed) + .put("lastName", entity.lastName) + .put("failed", entity.failed) + .put("notes", entity.notes); + } - @Override - public void read(NitriteMapper mapper, Document document) { - firstName = document.get("firstName", String.class); - processed = document.get("processed", Boolean.class); - lastName = document.get("lastName", String.class); - failed = document.get("failed", Boolean.class); - notes = document.get("notes", String.class); + @Override + public StressRecord fromDocument(Document document, NitriteMapper nitriteMapper) { + StressRecord entity = new StressRecord(); + entity.firstName = document.get("firstName", String.class); + entity.processed = document.get("processed", Boolean.class); + entity.lastName = document.get("lastName", String.class); + entity.failed = document.get("failed", Boolean.class); + entity.notes = document.get("notes", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java index 8eaa3faf1..ddea32812 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.Date; @@ -30,7 +30,7 @@ * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class SubEmployee implements Mappable { +public class SubEmployee { @Getter @Setter private Long empId; @@ -43,18 +43,28 @@ public class SubEmployee implements Mappable { @Setter private String address; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("empId", empId) - .put("joinDate", joinDate) - .put("address", address); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return SubEmployee.class; + } + + @Override + public Document toDocument(SubEmployee entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("empId", entity.empId) + .put("joinDate", entity.joinDate) + .put("address", entity.address); + } - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - joinDate = document.get("joinDate", Date.class); - address = document.get("address", String.class); + @Override + public SubEmployee fromDocument(Document document, NitriteMapper nitriteMapper) { + SubEmployee entity = new SubEmployee(); + entity.empId = document.get("empId", Long.class); + entity.joinDate = document.get("joinDate", Date.class); + entity.address = document.get("address", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java index 198f16d0c..523cebb87 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java @@ -19,10 +19,7 @@ import lombok.Getter; import lombok.Setter; -import org.dizitart.no2.collection.Document; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Index; /** @@ -30,17 +27,7 @@ */ @Getter @Setter -@Index(value = "text", type = IndexType.FULL_TEXT) -public class SuperDuperClass implements Mappable { +@Index(fields = "text", type = IndexType.FULL_TEXT) +public class SuperDuperClass { private String text; - - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("text", text); - } - - @Override - public void read(NitriteMapper mapper, Document document) { - text = document.get("text", String.class); - } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java index 79cafe0dc..bdeef8744 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,20 +29,29 @@ */ @Getter @Setter -public class WithClassField implements Mappable { +public class WithClassField { @Id private String name; private Class clazz; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("clazz", clazz); - } + public static class Converter implements EntityConverter { + @Override + public Class getEntityType() { + return WithClassField.class; + } + + @Override + public Document toDocument(WithClassField entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("clazz", entity.clazz); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - clazz = document.get("clazz", Class.class); + @Override + public WithClassField fromDocument(Document document, NitriteMapper nitriteMapper) { + WithClassField entity = new WithClassField(); + entity.name = document.get("name", String.class); + entity.clazz = document.get("clazz", Class.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java index 229f1aaf7..125e94b05 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java @@ -21,7 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.util.Date; @@ -32,19 +32,29 @@ @Getter @Setter @EqualsAndHashCode -public class WithDateId implements Mappable { +public class WithDateId { private Date id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("id", id); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithDateId.class; + } + + @Override + public Document toDocument(WithDateId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("id", entity.id); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - id = document.get("id", Date.class); + @Override + public WithDateId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithDateId entity = new WithDateId(); + entity.name = document.get("name", String.class); + entity.id = document.get("id", Date.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java index a0048a6bb..15792f12e 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,17 +29,27 @@ */ @Getter @Setter -public class WithEmptyStringId implements Mappable { +public class WithEmptyStringId { @Id private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithEmptyStringId.class; + } + + @Override + public Document toDocument(WithEmptyStringId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); + @Override + public WithEmptyStringId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithEmptyStringId entity = new WithEmptyStringId(); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java index 2364be76d..6748256c6 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java @@ -20,7 +20,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteId; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -28,21 +28,31 @@ * @author Anindya Chatterjee */ @Data -public class WithNitriteId implements Mappable { +public class WithNitriteId { @Id public NitriteId idField; public String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("idField", idField) - .put("name", name); - } + public static class WithNitriteIdConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithNitriteId.class; + } + + @Override + public Document toDocument(WithNitriteId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("idField", entity.idField) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - idField = document.get("idField", NitriteId.class); - name = document.get("name", String.class); + @Override + public WithNitriteId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithNitriteId entity = new WithNitriteId(); + entity.idField = document.get("idField", NitriteId.class); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java index 0dc3fb68b..dfac35bfa 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,21 +29,31 @@ */ @Getter @Setter -public class WithNullId implements Mappable { +public class WithNullId { @Id private String name; - private long number; + private Long number; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithNullId.class; + } + + @Override + public Document toDocument(WithNullId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithNullId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithNullId entity = new WithNullId(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java index 5f00f97d4..634a90fb8 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,17 +29,27 @@ */ @Getter @Setter -public class WithObjectId implements Mappable { +public class WithObjectId { @Id private WithOutId withOutId; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("withOutId", withOutId); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithObjectId.class; + } + + @Override + public Document toDocument(WithObjectId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("withOutId", entity.withOutId); + } - @Override - public void read(NitriteMapper mapper, Document document) { - withOutId = document.get("withOutId", WithOutId.class); + @Override + public WithObjectId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithObjectId entity = new WithObjectId(); + entity.withOutId = document.get("withOutId", WithOutId.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java index a7a2c56c9..e6783a89e 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java @@ -19,32 +19,41 @@ import lombok.EqualsAndHashCode; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class WithOutGetterSetter implements Mappable { +public class WithOutGetterSetter { private String name; - private long number; + private Long number; public WithOutGetterSetter() { name = "test"; - number = 2; + number = 2L; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("number", number); + public static class Converter implements EntityConverter { - } + @Override + public Class getEntityType() { + return WithOutGetterSetter.class; + } + + @Override + public Document toDocument(WithOutGetterSetter entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithOutGetterSetter fromDocument(Document document, NitriteMapper nitriteMapper) { + WithOutGetterSetter entity = new WithOutGetterSetter(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java index 09b63c2c2..e34e41702 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java @@ -20,33 +20,45 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import java.io.Serializable; + /** * @author Anindya Chatterjee. */ @Getter @Setter -public class WithOutId implements Comparable, Mappable { +public class WithOutId implements Comparable, Serializable { private String name; - private long number; + private Long number; @Override public int compareTo(WithOutId o) { return Long.compare(number, o.number); } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public Class getEntityType() { + return WithOutId.class; + } + + @Override + public Document toDocument(WithOutId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("name", entity.name) + .put("number", entity.number); + } + + @Override + public WithOutId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithOutId entity = new WithOutId(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java index a425209fb..de08cf19b 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java @@ -19,38 +19,47 @@ import lombok.EqualsAndHashCode; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; /** * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class WithPrivateConstructor implements Mappable { +public class WithPrivateConstructor { private String name; - private long number; + private Long number; private WithPrivateConstructor() { name = "test"; - number = 2; + number = 2L; } - public static WithPrivateConstructor create(final String name, final long number) { + public static WithPrivateConstructor create(final String name, final Long number) { WithPrivateConstructor obj = new WithPrivateConstructor(); obj.number = number; obj.name = name; return obj; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithPrivateConstructor.class; + } + + @Override + public Document toDocument(WithPrivateConstructor entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithPrivateConstructor fromDocument(Document document, NitriteMapper nitriteMapper) { + String name = document.get("name", String.class); + Long number = document.get("number", Long.class); + return WithPrivateConstructor.create(name, number); + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java index 793388231..81b69a0c1 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java @@ -18,27 +18,37 @@ package org.dizitart.no2.integration.repository.data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; /** * @author Anindya Chatterjee. */ -public class WithPublicField implements Mappable { +public class WithPublicField { @Id public String name; - public long number; + public Long number; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("name", name) - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithPublicField.class; + } + + @Override + public Document toDocument(WithPublicField entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.name) + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - name = document.get("name", String.class); - number = document.get("number", Long.class); + @Override + public WithPublicField fromDocument(Document document, NitriteMapper nitriteMapper) { + WithPublicField entity = new WithPublicField(); + entity.name = document.get("name", String.class); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java index 9c1ade657..76ea86243 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -29,19 +29,29 @@ */ @Getter @Setter -public class WithTransientField implements Mappable { +public class WithTransientField { private transient String name; @Id - private long number; + private Long number; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("number", number); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return WithTransientField.class; + } + + @Override + public Document toDocument(WithTransientField entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("number", entity.number); + } - @Override - public void read(NitriteMapper mapper, Document document) { - number = document.get("number", Long.class); + @Override + public WithTransientField fromDocument(Document document, NitriteMapper nitriteMapper) { + WithTransientField entity = new WithTransientField(); + entity.number = document.get("number", Long.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java index 9ecb9a68b..9c721c80c 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java @@ -19,7 +19,7 @@ import lombok.Data; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -27,39 +27,60 @@ * @author Anindya Chatterjee */ @Data -public class WithoutEmbeddedId implements Mappable { +public class WithoutEmbeddedId { @Id private NestedId nestedId; private String data; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("nestedId", nestedId.write(mapper)) - .put("data", data); - } + @Data + public static class NestedId { + private Long id; + + public static class Converter implements EntityConverter { - @Override - public void read(NitriteMapper mapper, Document document) { - Document nestedId = document.get("nestedId", Document.class); - this.nestedId = mapper.convert(nestedId, NestedId.class); - this.data = document.get("data", String.class); + @Override + public Class getEntityType() { + return NestedId.class; + } + + @Override + public Document toDocument(NestedId entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("id", entity.id); + } + + @Override + public NestedId fromDocument(Document document, NitriteMapper nitriteMapper) { + NestedId entity = new NestedId(); + entity.id = document.get("id", Long.class); + return entity; + } + } } + public static class Converter implements EntityConverter { - @Data - public static class NestedId implements Mappable { - private Long id; + @Override + public Class getEntityType() { + return WithoutEmbeddedId.class; + } @Override - public Document write(NitriteMapper mapper) { + public Document toDocument(WithoutEmbeddedId entity, NitriteMapper nitriteMapper) { return Document.createDocument() - .put("id", id); + .put("nestedId", nitriteMapper.convert(entity.nestedId, Document.class)) + .put("data", entity.data); } @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); + public WithoutEmbeddedId fromDocument(Document document, NitriteMapper nitriteMapper) { + WithoutEmbeddedId entity = new WithoutEmbeddedId(); + Document nestedId = document.get("nestedId", Document.class); + + entity.nestedId = nitriteMapper.convert(nestedId, NestedId.class); + entity.data = document.get("data", String.class); + + return entity; } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java new file mode 100644 index 000000000..c6bfe8bb9 --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; + +@Data +public class Manufacturer { + private String name; + private String address; + private Integer uniqueId; +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java new file mode 100644 index 000000000..f31d7b0bb --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ManufacturerConverter implements EntityConverter { + @Override + public Class getEntityType() { + return Manufacturer.class; + } + + @Override + public Document toDocument(Manufacturer entity, NitriteMapper nitriteMapper) { + return Document.createDocument("name", entity.getName()) + .put("address", entity.getAddress()) + .put("uniqueId", entity.getUniqueId()); + } + + @Override + public Manufacturer fromDocument(Document document, NitriteMapper nitriteMapper) { + Manufacturer manufacturer = new Manufacturer(); + manufacturer.setName(document.get("name", String.class)); + manufacturer.setAddress(document.get("address", String.class)); + manufacturer.setUniqueId(document.get("uniqueId", Integer.class)); + return manufacturer; + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java new file mode 100644 index 000000000..8d066491a --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.repository.EntityIndex; +import org.dizitart.no2.repository.EntityDecorator; +import org.dizitart.no2.repository.EntityId; + +import java.util.List; + +public class ManufacturerDecorator implements EntityDecorator { + @Override + public Class getEntityType() { + return Manufacturer.class; + } + + @Override + public EntityId getIdField() { + return null; + } + + @Override + public List getIndexFields() { + return null; + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java new file mode 100644 index 000000000..89b332c8a --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +@Data +public class MiniProduct { + private String uniqueId; + private String manufacturerName; + private Double price; + + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return MiniProduct.class; + } + + @Override + public Document toDocument(MiniProduct entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("productId.uniqueId", entity.getUniqueId()) + .put("manufacturer.name", entity.getManufacturerName()) + .put("price", entity.getPrice()); + } + + @Override + public MiniProduct fromDocument(Document document, NitriteMapper nitriteMapper) { + MiniProduct entity = new MiniProduct(); + entity.setUniqueId(document.get("productId.uniqueId", String.class)); + entity.setManufacturerName(document.get("manufacturer.name", String.class)); + entity.setPrice(document.get("price", Double.class)); + return entity; + } + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java new file mode 100644 index 000000000..ff59219f9 --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; + +@Data +public class Product { + private ProductId productId; + private Manufacturer manufacturer; + private String productName; + private Double price; +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java new file mode 100644 index 000000000..0421c0cac --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ProductConverter implements EntityConverter { + @Override + public Class getEntityType() { + return Product.class; + } + + @Override + public Document toDocument(Product entity, NitriteMapper nitriteMapper) { + Document productId = nitriteMapper.convert(entity.getProductId(), Document.class); + Document manufacturer = nitriteMapper.convert(entity.getManufacturer(), Document.class); + + return Document.createDocument() + .put("productId", productId) + .put("manufacturer", manufacturer) + .put("productName", entity.getProductName()) + .put("price", entity.getPrice()); + } + + @Override + public Product fromDocument(Document document, NitriteMapper nitriteMapper) { + Product entity = new Product(); + ProductId productId = nitriteMapper.convert(document.get("productId", Document.class), ProductId.class); + Manufacturer manufacturer = nitriteMapper.convert(document.get("manufacturer", Document.class), + Manufacturer.class); + entity.setProductId(productId); + entity.setManufacturer(manufacturer); + entity.setProductName(document.get("productName", String.class)); + entity.setPrice(document.get("price", Double.class)); + return entity; + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java new file mode 100644 index 000000000..180296343 --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.repository.EntityIndex; +import org.dizitart.no2.index.IndexType; +import org.dizitart.no2.repository.EntityDecorator; +import org.dizitart.no2.repository.EntityId; + +import java.util.Arrays; +import java.util.List; + +public class ProductDecorator implements EntityDecorator { + @Override + public Class getEntityType() { + return Product.class; + } + + @Override + public EntityId getIdField() { + return new EntityId("productId", "uniqueId", "productCode"); + } + + @Override + public List getIndexFields() { + return Arrays.asList( + new EntityIndex(IndexType.NON_UNIQUE, "manufacturer.name"), + new EntityIndex(IndexType.UNIQUE, "productName", "manufacturer.uniqueId") + ); + } + + @Override + public String getEntityName() { + return "product"; + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java new file mode 100644 index 000000000..dd2f9f04e --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import lombok.Data; + +@Data +public class ProductId { + private String uniqueId; + private String productCode; +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java new file mode 100644 index 000000000..2c02fddc0 --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.repository.decorator; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; + +public class ProductIdConverter implements EntityConverter { + @Override + public Class getEntityType() { + return ProductId.class; + } + + @Override + public Document toDocument(ProductId entity, NitriteMapper nitriteMapper) { + return Document.createDocument("uniqueId", entity.getUniqueId()) + .put("productCode", entity.getProductCode()); + } + + @Override + public ProductId fromDocument(Document document, NitriteMapper nitriteMapper) { + ProductId entity = new ProductId(); + entity.setUniqueId(document.get("uniqueId", String.class)); + entity.setProductCode(document.get("productCode", String.class)); + return entity; + } +} diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java index 05a2b095b..931f70a14 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java @@ -21,7 +21,7 @@ import org.dizitart.no2.integration.collection.BaseCollectionTest; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.TransactionException; import org.dizitart.no2.index.IndexType; @@ -299,11 +299,11 @@ public void testRollbackClear() { txCol.insert(document2); collection.insert(document2); - throw new TransactionException("failed"); + transaction.commit(); } catch (TransactionException e) { assert transaction != null; transaction.rollback(); - assertEquals(2, collection.size()); + assertEquals(0, collection.size()); } } } @@ -431,7 +431,7 @@ public void testCommitDropCollection() { boolean expectedException = false; try { assertEquals(0, txCol.size()); - } catch (NitriteIOException e) { + } catch (TransactionException e) { expectedException = true; } assertTrue(expectedException); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java index 5e93303ed..f1107695f 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java @@ -22,7 +22,7 @@ import org.dizitart.no2.integration.repository.data.SubEmployee; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; -import org.dizitart.no2.collection.meta.Attributes; +import org.dizitart.no2.common.meta.Attributes; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.TransactionException; import org.dizitart.no2.index.IndexType; @@ -324,11 +324,10 @@ public void testRollbackClear() { repository.insert(txData2); transaction.commit(); - fail(); } catch (TransactionException e) { assert transaction != null; transaction.rollback(); - assertEquals(2, repository.size()); + assertEquals(0, repository.size()); } } } @@ -454,7 +453,7 @@ public void testCommitDropRepository() { boolean expectedException = false; try { assertEquals(0, txRepo.size()); - } catch (NitriteIOException e) { + } catch (TransactionException e) { expectedException = true; } assertTrue(expectedException); diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java index 1e5ed7ca6..104ccd4ad 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/transaction/TxData.java @@ -21,7 +21,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; @@ -31,20 +31,30 @@ @Data @NoArgsConstructor @AllArgsConstructor -class TxData implements Mappable { +public class TxData { @Id private Long id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id) - .put("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return TxData.class; + } + + @Override + public Document toDocument(TxData entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Long.class); - name = document.get("name", String.class); + @Override + public TxData fromDocument(Document document, NitriteMapper nitriteMapper) { + TxData entity = new TxData(); + entity.id = document.get("id", Long.class); + entity.name = document.get("name", String.class); + return entity; + } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/GithubIssues.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/GithubIssues.java index d5e187d63..d2127df26 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/GithubIssues.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/GithubIssues.java @@ -20,8 +20,9 @@ import lombok.Data; import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.repository.ObjectRepository; import org.junit.Test; @@ -34,10 +35,14 @@ public void testIssue412() { RocksDBModule dbModule = RocksDBModule.withConfig() .filePath("/tmp/rocks-demo") .build(); + Nitrite db = Nitrite.builder() .loadModule(dbModule) .openOrCreate(); + SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper(); + documentMapper.registerEntityConverter(new TestData.Converter()); + // Step 1 // NitriteCollection collection = db.getCollection("test"); // Document document = Document.createDocument("a", 1).put("b", 2); @@ -54,19 +59,30 @@ public void testIssue412() { } @Data - public static class TestData implements Mappable { + public static class TestData { private Integer id; private String name; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("id", id).put("name", name); - } + public static class Converter implements EntityConverter { + + @Override + public Class getEntityType() { + return TestData.class; + } + + @Override + public Document toDocument(TestData entity, NitriteMapper nitriteMapper) { + return Document.createDocument("id", entity.id) + .put("name", entity.name); + } - @Override - public void read(NitriteMapper mapper, Document document) { - id = document.get("id", Integer.class); - name = document.get("name", String.class); + @Override + public TestData fromDocument(Document document, NitriteMapper nitriteMapper) { + TestData entity = new TestData(); + entity.id = document.get("id", Integer.class); + entity.name = document.get("name", String.class); + return entity; + } } } } diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/RocksDBStoreTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/RocksDBStoreTest.java index d34087441..8fa98466b 100644 --- a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/RocksDBStoreTest.java +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/rocksdb/RocksDBStoreTest.java @@ -17,11 +17,11 @@ package org.dizitart.no2.rocksdb; -import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.NitriteException; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.*; public class RocksDBStoreTest { @@ -46,16 +46,20 @@ public void testOpenMap() { RocksDBStore rocksDBStore = new RocksDBStore(); rocksDBStore.setStoreConfig(rocksDBConfig); Class keyType = Object.class; - rocksDBStore.openMap("Map Name", keyType, Object.class); + rocksDBStore.openMap("MapName", keyType, Object.class); verify(rocksDBConfig).objectFormatter(); } - @Test + @Test(expected = NitriteException.class) public void testOpenRTree() { + RocksDBConfig rocksDBConfig = mock(RocksDBConfig.class); + when(rocksDBConfig.objectFormatter()).thenThrow(new NitriteException("An error occurred")); + RocksDBStore rocksDBStore = new RocksDBStore(); + rocksDBStore.setStoreConfig(rocksDBConfig); Class keyType = Object.class; - assertThrows(InvalidOperationException.class, - () -> rocksDBStore.openRTree("R Tree Name", keyType, Object.class)); + rocksDBStore.openRTree("Rtree", keyType, Object.class); + verify(rocksDBConfig).objectFormatter(); } } diff --git a/nitrite-spatial/build.gradle b/nitrite-spatial/build.gradle index b40a66772..6a743b422 100644 --- a/nitrite-spatial/build.gradle +++ b/nitrite-spatial/build.gradle @@ -48,9 +48,9 @@ dependencies { api "org.slf4j:slf4j-api" api "org.locationtech.jts:jts-core" - annotationProcessor "org.projectlombok:lombok:1.18.22" + annotationProcessor "org.projectlombok:lombok:1.18.24" - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" + testAnnotationProcessor "org.projectlombok:lombok:1.18.24" testImplementation project(path: ':nitrite-mvstore-adapter', configuration: 'default') testImplementation project(path: ':nitrite-jackson-mapper', configuration: 'default') testImplementation "junit:junit:4.13.2" diff --git a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/GeometryUtils.java b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/GeometryUtils.java new file mode 100644 index 000000000..0ec577d71 --- /dev/null +++ b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/GeometryUtils.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017-2022 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.spatial; + +import org.dizitart.no2.exceptions.NitriteIOException; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; +import org.locationtech.jts.io.WKTWriter; + +public class GeometryUtils { + private static final String GEOMETRY_ID = "geometry:"; + private static WKTWriter writer; + private static WKTReader reader; + + static { + writer = new WKTWriter(); + reader = new WKTReader(); + } + + private GeometryUtils() { + } + + public static String toString(Geometry geometry) { + return GEOMETRY_ID + writer.write(geometry); + } + + public static Geometry fromString(String geometryValue) { + try { + if (geometryValue.contains(GEOMETRY_ID)) { + String geometry = geometryValue.replace(GEOMETRY_ID, ""); + return reader.read(geometry); + } else { + throw new NitriteIOException("Not a valid WKT geometry string " + geometryValue); + } + } catch (ParseException pe) { + throw new NitriteIOException("Failed to parse WKT geometry string", pe); + } + } +} diff --git a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndex.java b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndex.java index 12c6b7087..303155ae8 100644 --- a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndex.java +++ b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndex.java @@ -113,14 +113,14 @@ public LinkedHashSet findNitriteIds(FindPlan findPlan) { if (indexScanFilter == null || indexScanFilter.getFilters() == null || indexScanFilter.getFilters().isEmpty()) { - throw new FilterException("no spatial filter found"); + throw new FilterException("No spatial filter found"); } List filters = indexScanFilter.getFilters(); ComparableFilter filter = filters.get(0); if (!(filter instanceof SpatialFilter)) { - throw new FilterException("spatial filter must be the first filter for index scan"); + throw new FilterException("Spatial filter must be the first filter for index scan"); } RecordStream keys = null; @@ -154,10 +154,10 @@ private NitriteRTree findIndexMap() { private Geometry parseGeometry(String field, Object fieldValue) { if (fieldValue == null) return null; if (fieldValue instanceof String) { - return nitriteConfig.nitriteMapper().convert(fieldValue, Geometry.class); + return GeometryUtils.fromString((String) fieldValue); } else if (fieldValue instanceof Geometry) { return (Geometry) fieldValue; } - throw new IndexingException("field " + field + " does not contain Geometry data"); + throw new IndexingException("Field " + field + " does not contain Geometry data"); } } diff --git a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndexer.java b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndexer.java index 501a394d4..f39c50a1f 100644 --- a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndexer.java +++ b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/SpatialIndexer.java @@ -57,7 +57,7 @@ public String getIndexType() { @Override public void validateIndex(Fields fields) { if (fields.getFieldNames().size() > 1) { - throw new IndexingException("spatial index can only be created on a single field"); + throw new IndexingException("Spatial index can only be created on a single field"); } } diff --git a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryDeserializer.java b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryDeserializer.java index bb58df1fd..e9288f867 100644 --- a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryDeserializer.java +++ b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryDeserializer.java @@ -20,14 +20,11 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer; import lombok.extern.slf4j.Slf4j; +import org.dizitart.no2.spatial.GeometryUtils; import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.io.ParseException; -import org.locationtech.jts.io.WKTReader; import java.io.IOException; -import static org.dizitart.no2.spatial.mapper.GeometryExtension.GEOMETRY_ID; - /** * @author Anindya Chatterjee */ @@ -41,17 +38,6 @@ protected GeometryDeserializer() { @Override public Geometry deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { String value = p.getValueAsString(); - WKTReader reader = new WKTReader(); - try { - if (value.contains(GEOMETRY_ID)) { - String geometry = value.replace(GEOMETRY_ID, ""); - return reader.read(geometry); - } else { - throw new ParseException("Not a valid geometry value " + value); - } - } catch (ParseException e) { - log.error("Error while parsing WKT geometry string", e); - throw new IOException(e); - } + return GeometryUtils.fromString(value); } } diff --git a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryExtension.java b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryExtension.java deleted file mode 100644 index 23a7b7c51..000000000 --- a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryExtension.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2017-2020. Nitrite author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dizitart.no2.spatial.mapper; - -import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.module.SimpleModule; -import org.dizitart.no2.common.mapper.JacksonExtension; -import org.locationtech.jts.geom.Geometry; - -import java.util.List; - -import static org.dizitart.no2.common.util.Iterables.listOf; - -/** - * Class that registers capability of serializing {@code Geometry} objects with the Jackson core. - * - * @author Anindya Chatterjee - * @since 4.0.0 - */ -public class GeometryExtension implements JacksonExtension { - /** - * The constant GEOMETRY_ID - */ - public static final String GEOMETRY_ID = "geometry:"; - - @Override - public List> getSupportedTypes() { - return listOf(Geometry.class); - } - - @Override - public Module getModule() { - return new SimpleModule() { - @Override - public void setupModule(SetupContext context) { - addSerializer(Geometry.class, new GeometrySerializer()); - addDeserializer(Geometry.class, new GeometryDeserializer()); - super.setupModule(context); - } - }; - } -} diff --git a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateSocketListener.java b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryModule.java similarity index 55% rename from nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateSocketListener.java rename to nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryModule.java index 5df52b353..74ffac9be 100644 --- a/nitrite-replication/src/main/java/org/dizitart/no2/sync/net/DataGateSocketListener.java +++ b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometryModule.java @@ -14,40 +14,23 @@ * limitations under the License. */ -package org.dizitart.no2.sync.net; +package org.dizitart.no2.spatial.mapper; -import okhttp3.Response; -import okio.ByteString; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.locationtech.jts.geom.Geometry; /** + * Class that registers capability of serializing {@link Geometry} objects with the Jackson core. + * * @author Anindya Chatterjee + * @since 4.0.0 */ -public interface DataGateSocketListener { - default void onOpen(Response response) { - - } - - default void onMessage(String text) { - - } - - default void onMessage(ByteString bytes) { - - } - - default void onReconnect() { - - } - - default void onClosing(int code, String reason) { - - } - - default void onClosed(int code, String reason) { - - } - - default void onFailure(Throwable error, Response response) { +public class GeometryModule extends SimpleModule { + @Override + public void setupModule(SetupContext context) { + addSerializer(Geometry.class, new GeometrySerializer()); + addDeserializer(Geometry.class, new GeometryDeserializer()); + super.setupModule(context); } } diff --git a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometrySerializer.java b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometrySerializer.java index 62585bdf8..dbb9c7b44 100644 --- a/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometrySerializer.java +++ b/nitrite-spatial/src/main/java/org/dizitart/no2/spatial/mapper/GeometrySerializer.java @@ -19,13 +19,11 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdScalarSerializer; +import org.dizitart.no2.spatial.GeometryUtils; import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.io.WKTWriter; import java.io.IOException; -import static org.dizitart.no2.spatial.mapper.GeometryExtension.GEOMETRY_ID; - /** * @author Anindya Chatterjee */ @@ -38,9 +36,7 @@ protected GeometrySerializer() { @Override public void serialize(Geometry value, JsonGenerator gen, SerializerProvider provider) throws IOException { if (value != null) { - WKTWriter writer = new WKTWriter(); - String wktString = writer.write(value); - gen.writeString(GEOMETRY_ID + wktString); + gen.writeString(GeometryUtils.toString(value)); } } } diff --git a/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/SpatialData.java b/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/SpatialData.java index 620fc2874..5ed1a120e 100644 --- a/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/SpatialData.java +++ b/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/SpatialData.java @@ -27,7 +27,7 @@ * @author Anindya Chatterjee */ @Data -@Index(value = "geometry", type = SPATIAL_INDEX) +@Index(fields = "geometry", type = SPATIAL_INDEX) public class SpatialData { @Id private Long id; diff --git a/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/TestUtil.java b/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/TestUtil.java index 99b0526f5..b131e2f47 100644 --- a/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/TestUtil.java +++ b/nitrite-spatial/src/test/java/org/dizitart/no2/spatial/TestUtil.java @@ -21,7 +21,7 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.common.mapper.JacksonMapperModule; import org.dizitart.no2.mvstore.MVStoreModule; -import org.dizitart.no2.spatial.mapper.GeometryExtension; +import org.dizitart.no2.spatial.mapper.GeometryModule; import java.io.File; import java.nio.file.Files; @@ -50,7 +50,7 @@ public static Nitrite createDb(String fileName) { return Nitrite.builder() .loadModule(module) - .loadModule(new JacksonMapperModule(new GeometryExtension())) + .loadModule(new JacksonMapperModule(new GeometryModule())) .loadModule(new SpatialModule()) .fieldSeparator(".") .openOrCreate(); diff --git a/nitrite-support/build.gradle b/nitrite-support/build.gradle index 5adece14b..f1948e232 100644 --- a/nitrite-support/build.gradle +++ b/nitrite-support/build.gradle @@ -47,11 +47,12 @@ dependencies { api "com.fasterxml.jackson.core:jackson-databind" api "commons-codec:commons-codec" - annotationProcessor "org.projectlombok:lombok:1.18.22" + annotationProcessor "org.projectlombok:lombok:1.18.24" - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" + testAnnotationProcessor "org.projectlombok:lombok:1.18.24" testImplementation project(path: ':nitrite-mvstore-adapter', configuration: 'default') testImplementation "junit:junit:4.13.2" + testImplementation "com.github.javafaker:javafaker:1.0.2" } test { diff --git a/nitrite-support/src/main/java/org/dizitart/no2/support/Exporter.java b/nitrite-support/src/main/java/org/dizitart/no2/support/Exporter.java index d3ee617a2..e8c35e301 100644 --- a/nitrite-support/src/main/java/org/dizitart/no2/support/Exporter.java +++ b/nitrite-support/src/main/java/org/dizitart/no2/support/Exporter.java @@ -113,7 +113,7 @@ public void exportTo(File file) { } File parent = file.getParentFile(); - // if parent dir does not exists, try to create it + // if parent dir does not exist, try to create it if (!parent.exists()) { boolean result = parent.mkdirs(); if (!result) { @@ -160,7 +160,7 @@ public void exportTo(Writer writer) { try { jsonExporter.exportData(); } catch (IOException | ClassNotFoundException e) { - throw new NitriteIOException("error while exporting data", e); + throw new NitriteIOException("Error while exporting data", e); } } } diff --git a/nitrite-support/src/main/java/org/dizitart/no2/support/Importer.java b/nitrite-support/src/main/java/org/dizitart/no2/support/Importer.java index cdfe968fe..20b8f52ef 100644 --- a/nitrite-support/src/main/java/org/dizitart/no2/support/Importer.java +++ b/nitrite-support/src/main/java/org/dizitart/no2/support/Importer.java @@ -116,7 +116,7 @@ public void importFrom(Reader reader) { try { jsonImporter.importData(); } catch (IOException | ClassNotFoundException e) { - throw new NitriteIOException("error while importing data", e); + throw new NitriteIOException("Error while importing data", e); } } } diff --git a/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonExporter.java b/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonExporter.java index cb2fcff73..2ed1ff5f7 100644 --- a/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonExporter.java +++ b/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonExporter.java @@ -60,7 +60,7 @@ public void exportData() throws IOException, ClassNotFoundException { if (collections.isEmpty()) { collectionNames = db.listCollectionNames(); repositoryNames = db.listRepositories(); - keyedRepositoryNames = db.listKeyedRepository(); + keyedRepositoryNames = db.listKeyedRepositories(); } else { collectionNames = new HashSet<>(); repositoryNames = new HashSet<>(); @@ -220,7 +220,7 @@ private String writeEncodedObject(Object object) { return Hex.encodeHexString(data); } } catch (IOException e) { - throw new NitriteIOException("failed to write object", e); + throw new NitriteIOException("Failed to write object", e); } } } diff --git a/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonImporter.java b/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonImporter.java index e62526647..059b21c5c 100644 --- a/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonImporter.java +++ b/nitrite-support/src/main/java/org/dizitart/no2/support/NitriteJsonImporter.java @@ -181,8 +181,8 @@ private void readIndices(PersistentCollection collection) throws IOException String data = parser.readValueAs(String.class); IndexDescriptor index = (IndexDescriptor) readEncodedObject(data); if (index != null) { - String[] fieldNames = index.getIndexFields().getFieldNames().toArray(new String[0]); - if (collection != null && index.getIndexFields() != null && !collection.hasIndex(fieldNames)) { + String[] fieldNames = index.getFields().getFieldNames().toArray(new String[0]); + if (collection != null && index.getFields() != null && !collection.hasIndex(fieldNames)) { collection.createIndex(indexOptions(index.getIndexType()), fieldNames); } } @@ -227,7 +227,7 @@ private Object readEncodedObject(String hexString) { } } } catch (Exception e) { - throw new NitriteIOException("error while reading data", e); + throw new NitriteIOException("Error while reading data", e); } } } diff --git a/nitrite-support/src/test/java/org/dizitart/no2/support/BaseExternalTest.java b/nitrite-support/src/test/java/org/dizitart/no2/support/BaseExternalTest.java index 49055ade6..98a84d3a5 100644 --- a/nitrite-support/src/test/java/org/dizitart/no2/support/BaseExternalTest.java +++ b/nitrite-support/src/test/java/org/dizitart/no2/support/BaseExternalTest.java @@ -19,6 +19,7 @@ import org.dizitart.no2.Nitrite; import org.dizitart.no2.collection.Document; import org.dizitart.no2.collection.NitriteCollection; +import org.dizitart.no2.common.mapper.SimpleDocumentMapper; import org.dizitart.no2.mvstore.MVStoreModule; import org.dizitart.no2.repository.ObjectRepository; import org.junit.After; @@ -33,6 +34,7 @@ import java.util.UUID; import static org.dizitart.no2.common.Constants.*; +import static org.dizitart.no2.common.util.Iterables.setOf; import static org.junit.Assert.assertTrue; /** @@ -59,7 +61,7 @@ public static String getRandomTempDbFile() { if (!file.exists()) { assertTrue(file.mkdirs()); } - return file.getPath() + File.separator + UUID.randomUUID().toString() + ".db"; + return file.getPath() + File.separator + UUID.randomUUID() + ".db"; } @Before @@ -108,8 +110,14 @@ private Nitrite createDb(String filePath) { .filePath(filePath) .build(); + SimpleDocumentMapper documentMapper = new SimpleDocumentMapper(); + documentMapper.registerEntityConverter(new Employee.EmployeeConverter()); + documentMapper.registerEntityConverter(new Company.CompanyConverter()); + documentMapper.registerEntityConverter(new Note.NoteConverter()); + return Nitrite.builder() .loadModule(storeModule) + .loadModule(() -> setOf(documentMapper)) .fieldSeparator(".") .openOrCreate(); } diff --git a/nitrite-support/src/test/java/org/dizitart/no2/support/Company.java b/nitrite-support/src/test/java/org/dizitart/no2/support/Company.java index b92a9d6fe..d66f8254f 100644 --- a/nitrite-support/src/test/java/org/dizitart/no2/support/Company.java +++ b/nitrite-support/src/test/java/org/dizitart/no2/support/Company.java @@ -16,16 +16,13 @@ package org.dizitart.no2.support; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import lombok.Data; import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; +import org.dizitart.no2.common.mapper.NitriteMapper; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; import org.dizitart.no2.repository.annotations.Indices; -import org.dizitart.no2.common.mapper.Mappable; -import org.dizitart.no2.common.mapper.NitriteMapper; import java.io.Serializable; import java.util.Date; @@ -35,49 +32,57 @@ /** * @author Anindya Chatterjee. */ -@ToString -@EqualsAndHashCode +@Data @Indices({ - @Index(value = "companyName") + @Index(fields = "companyName") }) -public class Company implements Serializable, Mappable { - @Id - @Getter - @Setter +public class Company implements Serializable { + @Id(fieldName = "company_id") private Long companyId; - @Getter - @Setter private String companyName; - @Getter - @Setter private Date dateCreated; - @Getter - @Setter private List departments; - @Getter - @Setter private Map> employeeRecord; @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument("companyId", companyId) - .put("companyName", companyName) - .put("dateCreated", dateCreated) - .put("departments", departments) - .put("employeeRecord", employeeRecord); + public String toString() { + return "Company{" + + "companyId=" + companyId + + ", companyName='" + companyName + '\'' + + ", dateCreated=" + dateCreated + + ", departments=" + departments + + '}'; } - @Override - @SuppressWarnings("unchecked") - public void read(NitriteMapper mapper, Document document) { - companyId = document.get("companyId", Long.class); - companyName = document.get("companyName", String.class); - dateCreated = document.get("dateCreated", Date.class); - departments = document.get("departments", List.class); - employeeRecord = document.get("employeeRecord", Map.class); + public static class CompanyConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Company.class; + } + + @Override + public Document toDocument(Company entity, NitriteMapper nitriteMapper) { + return Document.createDocument("company_id", entity.companyId) + .put("companyName", entity.companyName) + .put("dateCreated", entity.dateCreated) + .put("departments", entity.departments) + .put("employeeRecord", entity.employeeRecord); + } + + @Override + public Company fromDocument(Document document, NitriteMapper nitriteMapper) { + Company entity = new Company(); + entity.companyId = document.get("company_id", Long.class); + entity.companyName = document.get("companyName", String.class); + entity.dateCreated = document.get("dateCreated", Date.class); + entity.departments = document.get("departments", List.class); + entity.employeeRecord = document.get("employeeRecord", Map.class); + return entity; + } } } diff --git a/nitrite-support/src/test/java/org/dizitart/no2/support/DataGenerator.java b/nitrite-support/src/test/java/org/dizitart/no2/support/DataGenerator.java index 53e52d1db..acea54ef5 100644 --- a/nitrite-support/src/test/java/org/dizitart/no2/support/DataGenerator.java +++ b/nitrite-support/src/test/java/org/dizitart/no2/support/DataGenerator.java @@ -16,13 +16,11 @@ package org.dizitart.no2.support; +import com.github.javafaker.Faker; import lombok.experimental.UtilityClass; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** @@ -32,12 +30,13 @@ public class DataGenerator { private static Random random = new Random(System.currentTimeMillis()); private static AtomicInteger counter = new AtomicInteger(random.nextInt()); + private static Faker faker = new Faker(); public static Company generateCompanyRecord() { Company company = new Company(); company.setCompanyId(System.nanoTime() + counter.incrementAndGet()); - company.setCompanyName(randomCompanyName()); - company.setDateCreated(randomDate()); + company.setCompanyName(faker.company().name()); + company.setDateCreated(faker.date().past(20 * 365, TimeUnit.DAYS)); List departments = departments(); company.setDepartments(departments); @@ -63,8 +62,8 @@ private static List generateEmployeeRecords(Company company, int count public static Employee generateEmployee() { Employee employee = new Employee(); employee.setEmpId(System.nanoTime() + counter.incrementAndGet()); - employee.setJoinDate(randomDate()); - employee.setAddress(UUID.randomUUID().toString().replace('-', ' ')); + employee.setJoinDate(faker.date().birthday()); + employee.setAddress(faker.address().fullAddress()); byte[] blob = new byte[random.nextInt(8000)]; random.nextBytes(blob); @@ -74,65 +73,18 @@ public static Employee generateEmployee() { return employee; } - private static Date randomDate() { - return new Date(-946771200000L + - (Math.abs(random.nextLong()) % (70L * 365 * 24 * 60 * 60 * 1000))); - } - public static Note randomNote() { - InputStream inputStream = ClassLoader.getSystemResourceAsStream("test.text"); - - try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { - String strLine; - long line = random.nextInt(49); - int count = 0; - while ((strLine = br.readLine()) != null) { - if (count == line) { - Note note = new Note(); - note.setNoteId(line); - note.setText(strLine); - return note; - } - count++; - } - } catch (IOException e) { - // ignore - } - // ignore - return null; - } - - private static String randomCompanyName() { - InputStream inputStream = ClassLoader.getSystemResourceAsStream("english.stop"); - - assert inputStream != null; - try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { - String strLine; - int line = random.nextInt(570); - int count = 0; - while ((strLine = br.readLine()) != null) { - if (count == line) { - return strLine + System.nanoTime() + " inc."; - } - count++; - } - } catch (IOException e) { - // ignore - } - // ignore - return null; + Note note = new Note(); + note.setNoteId((long) counter.incrementAndGet()); + note.setText(faker.lorem().paragraph()); + return note; } private static List departments() { - return new ArrayList() {{ - add("dev"); - add("hr"); - add("qa"); - add("dev-ops"); - add("sales"); - add("marketing"); - add("design"); - add("support"); - }}; + List departments = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + departments.add(faker.job().title()); + } + return departments; } } diff --git a/nitrite-support/src/test/java/org/dizitart/no2/support/Employee.java b/nitrite-support/src/test/java/org/dizitart/no2/support/Employee.java index 6e97fa636..c9841a7ea 100644 --- a/nitrite-support/src/test/java/org/dizitart/no2/support/Employee.java +++ b/nitrite-support/src/test/java/org/dizitart/no2/support/Employee.java @@ -21,11 +21,11 @@ import lombok.Setter; import lombok.ToString; import org.dizitart.no2.collection.Document; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Id; import org.dizitart.no2.repository.annotations.Index; import org.dizitart.no2.repository.annotations.Indices; -import org.dizitart.no2.common.mapper.Mappable; import org.dizitart.no2.common.mapper.NitriteMapper; import java.io.Serializable; @@ -37,11 +37,11 @@ @ToString @EqualsAndHashCode @Indices({ - @Index(value = "joinDate", type = IndexType.NON_UNIQUE), - @Index(value = "address", type = IndexType.FULL_TEXT), - @Index(value = "employeeNote.text", type = IndexType.FULL_TEXT) + @Index(fields = "joinDate", type = IndexType.NON_UNIQUE), + @Index(fields = "address", type = IndexType.FULL_TEXT), + @Index(fields = "employeeNote.text", type = IndexType.FULL_TEXT) }) -public class Employee implements Serializable, Mappable { +public class Employee implements Serializable { @Id @Getter @Setter @@ -79,26 +79,37 @@ public Employee(Employee copy) { employeeNote = copy.employeeNote; } - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument() - .put("empId", empId) - .put("joinDate", joinDate) - .put("address", address) - .put("blob", blob) - .put("employeeNote", employeeNote != null ? employeeNote.write(mapper) : null); - } + public static class EmployeeConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Employee.class; + } + + @Override + public Document toDocument(Employee entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("empId", entity.empId) + .put("joinDate", entity.joinDate) + .put("address", entity.address) + .put("blob", entity.blob) + .put("employeeNote", nitriteMapper.convert(entity.employeeNote, Document.class)); + } + + @Override + public Employee fromDocument(Document document, NitriteMapper nitriteMapper) { + Employee entity = new Employee(); - @Override - public void read(NitriteMapper mapper, Document document) { - empId = document.get("empId", Long.class); - joinDate = document.get("joinDate", Date.class); - address = document.get("address", String.class); - blob = document.get("blob", byte[].class); + entity.empId = document.get("empId", Long.class); + entity.joinDate = document.get("joinDate", Date.class); + entity.address = document.get("address", String.class); + entity.blob = document.get("blob", byte[].class); - if (document.get("employeeNote") != null) { - employeeNote = new Note(); - employeeNote.read(mapper, document.get("employeeNote", Document.class)); + if (document.get("employeeNote") != null) { + Document doc = document.get("employeeNote", Document.class); + entity.employeeNote = nitriteMapper.convert(doc, Note.class);; + } + return entity; } } } diff --git a/nitrite-support/src/test/java/org/dizitart/no2/support/Note.java b/nitrite-support/src/test/java/org/dizitart/no2/support/Note.java index 4f78a970b..ee3435060 100644 --- a/nitrite-support/src/test/java/org/dizitart/no2/support/Note.java +++ b/nitrite-support/src/test/java/org/dizitart/no2/support/Note.java @@ -20,7 +20,7 @@ import lombok.Getter; import lombok.Setter; import org.dizitart.no2.collection.Document; -import org.dizitart.no2.common.mapper.Mappable; +import org.dizitart.no2.common.mapper.EntityConverter; import org.dizitart.no2.common.mapper.NitriteMapper; import java.io.Serializable; @@ -29,7 +29,7 @@ * @author Anindya Chatterjee. */ @EqualsAndHashCode -public class Note implements Serializable, Mappable { +public class Note implements Serializable { @Getter @Setter private Long noteId; @@ -37,14 +37,26 @@ public class Note implements Serializable, Mappable { @Setter private String text; - @Override - public Document write(NitriteMapper mapper) { - return Document.createDocument().put("noteId", noteId).put("text", text); - } + public static class NoteConverter implements EntityConverter { + + @Override + public Class getEntityType() { + return Note.class; + } + + @Override + public Document toDocument(Note entity, NitriteMapper nitriteMapper) { + return Document.createDocument() + .put("noteId", entity.noteId) + .put("text", entity.text); + } - @Override - public void read(NitriteMapper mapper, Document document) { - noteId = document.get("noteId", Long.class); - text = document.get("text", String.class); + @Override + public Note fromDocument(Document document, NitriteMapper nitriteMapper) { + Note entity = new Note(); + entity.noteId = document.get("noteId", Long.class); + entity.text = document.get("text", String.class); + return entity; + } } } diff --git a/nitrite-support/src/test/resources/english.stop b/nitrite-support/src/test/resources/english.stop deleted file mode 100644 index 70572ebaa..000000000 --- a/nitrite-support/src/test/resources/english.stop +++ /dev/null @@ -1,571 +0,0 @@ -a -a's -able -about -above -according -accordingly -across -actually -after -afterwards -again -against -ain't -all -allow -allows -almost -alone -along -already -also -although -always -am -among -amongst -an -and -another -any -anybody -anyhow -anyone -anything -anyway -anyways -anywhere -apart -appear -appreciate -appropriate -are -aren't -around -as -aside -ask -asking -associated -at -available -away -awfully -b -be -became -because -become -becomes -becoming -been -before -beforehand -behind -being -believe -below -beside -besides -best -better -between -beyond -both -brief -but -by -c -c'mon -c's -came -can -can't -cannot -cant -cause -causes -certain -certainly -changes -clearly -co -com -come -comes -concerning -consequently -consider -considering -contain -containing -contains -corresponding -could -couldn't -course -currently -d -definitely -described -despite -did -didn't -different -do -does -doesn't -doing -don't -done -down -downwards -during -e -each -edu -eg -eight -either -else -elsewhere -enough -entirely -especially -et -etc -even -ever -every -everybody -everyone -everything -everywhere -ex -exactly -example -except -f -far -few -fifth -first -five -followed -following -follows -for -former -formerly -forth -four -from -further -furthermore -g -get -gets -getting -given -gives -go -goes -going -gone -got -gotten -greetings -h -had -hadn't -happens -hardly -has -hasn't -have -haven't -having -he -he's -hello -help -hence -her -here -here's -hereafter -hereby -herein -hereupon -hers -herself -hi -him -himself -his -hither -hopefully -how -howbeit -however -i -i'd -i'll -i'm -i've -ie -if -ignored -immediate -in -inasmuch -inc -indeed -indicate -indicated -indicates -inner -insofar -instead -into -inward -is -isn't -it -it'd -it'll -it's -its -itself -j -just -k -keep -keeps -kept -know -knows -known -l -last -lately -later -latter -latterly -least -less -lest -let -let's -like -liked -likely -little -look -looking -looks -ltd -m -mainly -many -may -maybe -me -mean -meanwhile -merely -might -more -moreover -most -mostly -much -must -my -myself -n -name -namely -nd -near -nearly -necessary -need -needs -neither -never -nevertheless -new -next -nine -no -nobody -non -none -noone -nor -normally -not -nothing -novel -now -nowhere -o -obviously -of -off -often -oh -ok -okay -old -on -once -one -ones -only -onto -or -other -others -otherwise -ought -our -ours -ourselves -out -outside -over -overall -own -p -particular -particularly -per -perhaps -placed -please -plus -possible -presumably -probably -provides -q -que -quite -qv -r -rather -rd -re -really -reasonably -regarding -regardless -regards -relatively -respectively -right -s -said -same -saw -say -saying -says -second -secondly -see -seeing -seem -seemed -seeming -seems -seen -self -selves -sensible -sent -serious -seriously -seven -several -shall -she -should -shouldn't -since -six -so -some -somebody -somehow -someone -something -sometime -sometimes -somewhat -somewhere -soon -sorry -specified -specify -specifying -still -sub -such -sup -sure -t -t's -take -taken -tell -tends -th -than -thank -thanks -thanx -that -that's -thats -the -their -theirs -them -themselves -then -thence -there -there's -thereafter -thereby -therefore -therein -theres -thereupon -these -they -they'd -they'll -they're -they've -think -third -this -thorough -thoroughly -those -though -three -through -throughout -thru -thus -to -together -too -took -toward -towards -tried -tries -truly -try -trying -twice -two -u -un -under -unfortunately -unless -unlikely -until -unto -up -upon -us -use -used -useful -uses -using -usually -uucp -v -value -various -very -via -viz -vs -w -want -wants -was -wasn't -way -we -we'd -we'll -we're -we've -welcome -well -went -were -weren't -what -what's -whatever -when -whence -whenever -where -where's -whereafter -whereas -whereby -wherein -whereupon -wherever -whether -which -while -whither -who -who's -whoever -whole -whom -whose -why -will -willing -wish -with -within -without -won't -wonder -would -would -wouldn't -x -y -yes -yet -you -you'd -you'll -you're -you've -your -yours -yourself -yourselves -z -zero \ No newline at end of file diff --git a/nitrite-support/src/test/resources/test.text b/nitrite-support/src/test/resources/test.text deleted file mode 100644 index c940fb50c..000000000 --- a/nitrite-support/src/test/resources/test.text +++ /dev/null @@ -1,50 +0,0 @@ -Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean massa nulla, elementum vitae velit nec, suscipit porttitor urna. Integer volutpat nibh et nisi congue ullamcorper. Cras laoreet consectetur massa a luctus. Etiam non sapien vitae enim semper elementum. Donec ullamcorper nibh quis magna vestibulum, eget faucibus sem posuere. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Ut sed nulla id risus ultricies facilisis. Curabitur scelerisque eros metus, in ultrices libero efficitur sit amet. Vivamus aliquet nec sapien sed vehicula. Praesent a mauris eget tortor consectetur euismod. Suspendisse tempus felis et ante dictum, nec eleifend tellus finibus. -Vivamus felis turpis, pretium id gravida ut, pharetra et massa. Duis lobortis velit in nibh tristique, sit amet tristique neque eleifend. Quisque et faucibus magna. Donec in enim odio. Vestibulum fringilla odio quis libero convallis, sed imperdiet ante scelerisque. Suspendisse rutrum, turpis eget mollis ultricies, purus ante pellentesque arcu, ut gravida libero elit eget odio. Nam quis congue purus. Sed commodo enim lacus, eu tristique nisi pretium id. Nulla diam odio, cursus a eleifend eu, tristique tempus dui. -Suspendisse dictum mollis justo. Aliquam tempus consectetur elit, in euismod erat tempor non. Nullam lacinia eget sapien quis luctus. Praesent odio nisl, dapibus sit amet pharetra ut, consequat et neque. Ut consequat, ligula at lacinia ultrices, erat nisl vehicula urna, at ullamcorper magna urna ut erat. Integer at orci fringilla, congue nibh et, lacinia lorem. Sed aliquet quis felis nec fringilla. Integer egestas vitae neque in consequat. In laoreet nisi risus, a pharetra mi placerat a. Etiam ultricies arcu non est volutpat fringilla. Ut et molestie augue, ac consequat urna. Nulla et egestas mi. Sed tempor tortor et velit lacinia volutpat. Etiam volutpat porta interdum. Aliquam vitae rutrum sem. -Phasellus venenatis nibh mauris, id lobortis lorem dictum quis. Cras at sagittis felis. Proin viverra neque et metus rhoncus suscipit. Nam aliquet aliquet est, eget convallis turpis hendrerit sit amet. Nam eget commodo justo. Morbi pharetra commodo sapien vitae luctus. Integer eu eleifend massa. Integer consectetur sapien nec ligula pellentesque, sit amet tempus lacus venenatis. Aenean efficitur risus sed ligula dapibus pellentesque. Sed consectetur est eget egestas ornare. -Integer sit amet iaculis orci. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Donec non dignissim nibh, sit amet ornare ipsum. Proin quis felis ut orci condimentum mattis a non urna. Aenean a metus nec lacus fringilla scelerisque sit amet id nisi. Nulla et velit in libero laoreet mollis. Nam malesuada leo non enim hendrerit facilisis a quis enim. -Nullam in diam ut ex mattis aliquet sed et ipsum. Morbi ut nunc et ex mollis convallis. Donec dui velit, iaculis at sapien ullamcorper, dictum maximus nibh. Maecenas porttitor arcu quis ex molestie elementum. Donec scelerisque ut erat at finibus. Nunc nec tempor mauris, sed posuere magna. In hac habitasse platea dictumst. Phasellus elit nisl, pellentesque vestibulum arcu quis, accumsan cursus nisi. Nunc eget orci non felis pretium vestibulum nec in mi. Nulla tincidunt, tortor id bibendum scelerisque, nibh libero lacinia felis, at pretium arcu lectus eu justo. Aenean ac elit id leo finibus vehicula. -Donec sed orci lobortis, posuere purus nec, molestie odio. Mauris elementum libero vel nibh ultrices vehicula. Quisque dignissim ipsum libero, eu elementum quam dignissim a. Mauris magna felis, tincidunt eu dolor vel, sollicitudin porta mi. Integer tempor volutpat leo. Nunc a neque lectus. Nullam pretium tempus congue. -Sed vehicula ipsum ipsum, et maximus augue gravida ut. Nunc augue lorem, volutpat ac volutpat nec, commodo ut ex. Duis dignissim est enim, non dapibus urna pharetra a. Aliquam vitae viverra nibh, ut ullamcorper nulla. Pellentesque ultricies aliquet magna, eget fermentum turpis. Etiam ut urna elementum, laoreet est ut, commodo quam. In quis libero vitae lacus scelerisque dignissim. Phasellus libero ligula, porttitor ac enim sed, ultrices accumsan erat. Quisque cursus tortor sit amet tellus laoreet, eu finibus velit laoreet. Duis luctus massa non nulla luctus, at imperdiet lacus condimentum. Ut nulla metus, elementum sed porta id, fringilla non enim. Fusce id rutrum ipsum. Etiam rhoncus finibus faucibus. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. -Etiam placerat aliquet nunc, id facilisis metus auctor at. Ut mi lacus, pharetra in egestas in, accumsan nec urna. Donec placerat erat quis enim vehicula, eget sollicitudin elit auctor. Curabitur ac auctor mauris. Nulla facilisi. Nam suscipit commodo mi, ut consequat neque finibus et. Fusce vestibulum porttitor odio, at tincidunt urna consectetur eu. Nullam lobortis risus sed convallis commodo. Cras non nibh vehicula, venenatis eros non, ultricies neque. Pellentesque enim nisl, elementum et libero id, laoreet dapibus odio. Aenean quis erat suscipit, mattis lectus quis, consectetur nibh. Vivamus vel sem egestas, ultricies mauris at, bibendum velit. Phasellus cursus elit erat, ut molestie risus condimentum vitae. Nam sapien mauris, ullamcorper sit amet dui id, accumsan lobortis dolor. Curabitur ultrices metus felis, ac sollicitudin eros vulputate at. Nunc eget tristique eros. -Vestibulum in lobortis nulla. Morbi mattis turpis ut tellus rutrum, at consectetur lacus mollis. Etiam maximus lectus ante, scelerisque aliquet turpis pharetra et. Fusce vulputate efficitur enim, at varius augue sagittis nec. Aenean volutpat volutpat metus, ut faucibus ex posuere dictum. Vivamus est enim, volutpat aliquam mi vel, dictum lobortis leo. Cras luctus fermentum tellus, sodales pellentesque lectus rutrum id. Sed in condimentum leo. Etiam ultricies porttitor ligula, et sodales purus imperdiet ac. Praesent non semper mi. Nullam bibendum auctor mi sit amet malesuada. Maecenas luctus accumsan ante feugiat egestas. Mauris tellus felis, venenatis sit amet risus nec, auctor auctor libero. Suspendisse purus risus, suscipit at lectus ut, maximus convallis nunc. Nulla facilisi. -Quisque commodo diam ut dui tristique luctus. Nam dignissim semper sollicitudin. Aenean condimentum orci ut augue suscipit, vitae ullamcorper urna ultrices. Nunc ac arcu eu massa placerat maximus. Cras ultrices elit id ipsum finibus, sit amet lacinia dui dignissim. Nulla suscipit suscipit libero non posuere. Aenean lobortis cursus metus, eu dignissim ligula pellentesque a. Donec viverra varius luctus. Mauris quis neque erat. Quisque pellentesque volutpat felis ultricies feugiat. -Fusce eu ante bibendum, tincidunt mauris bibendum, vulputate diam. Integer sit amet massa maximus, finibus ipsum in, scelerisque nunc. Quisque sodales neque quis tristique porttitor. Suspendisse molestie sed arcu nec commodo. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec gravida nunc at fringilla accumsan. Praesent id elit sem. Sed sagittis sed nunc sit amet volutpat. Suspendisse potenti. Etiam malesuada neque enim, a consectetur justo ultricies ac. -Aenean vel enim a diam aliquet tempor. Phasellus et blandit dui. Nulla facilisi. Vestibulum rhoncus dolor at risus vehicula, ut tincidunt arcu sodales. Duis at massa ut leo suscipit mollis hendrerit nec justo. Nullam ex erat, accumsan eu odio sit amet, ullamcorper hendrerit ligula. Nullam scelerisque ut turpis vitae imperdiet. Pellentesque fermentum urna id ex mollis, vel mattis leo interdum. -Aenean nec est ac est varius elementum. Vestibulum vulputate accumsan pellentesque. Sed pellentesque mi nec ipsum tincidunt, fringilla faucibus urna semper. Pellentesque quis consequat nisl. Sed viverra nulla sagittis ante blandit, nec vulputate purus dapibus. Nullam molestie nisi diam, vitae elementum massa pellentesque eu. Vestibulum lacinia tortor in nisi auctor, ut pharetra dui interdum. Mauris ac ante elit. -Praesent pharetra egestas sapien ut convallis. Pellentesque gravida maximus maximus. Aliquam tristique venenatis arcu, ut bibendum augue lacinia vel. Nulla vulputate sit amet nibh non vestibulum. Pellentesque et imperdiet ex, vel cursus dui. Integer facilisis, est et tincidunt cursus, ligula arcu faucibus tortor, id commodo orci orci in neque. Nam lacinia lectus tempus, finibus libero pretium, tempor nulla. Lorem ipsum dolor sit amet, consectetur adipiscing elit. -Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Sed massa ipsum, rhoncus pellentesque dui vestibulum, accumsan lacinia risus. Suspendisse ullamcorper enim vel sem blandit aliquet. Pellentesque bibendum erat eget quam pretium, sit amet cursus velit volutpat. Sed eget nunc at odio tristique venenatis. Mauris id sagittis quam, non aliquam magna. Curabitur sit amet neque a odio blandit consequat eu at est. Fusce in bibendum ipsum, quis fringilla justo. Nunc ut arcu non dui pretium vestibulum nec eu ipsum. Morbi ac dictum nulla. Vivamus vitae velit dolor. Duis quis aliquet felis. Praesent eu nulla vitae ex mattis imperdiet. -Aliquam finibus mauris arcu, non pellentesque diam interdum id. Praesent ornare lectus augue, eget lacinia odio venenatis a. Proin vulputate, nisl at feugiat euismod, massa arcu feugiat massa, a vehicula lorem nisi at diam. Aenean mattis, tellus sed hendrerit efficitur, purus purus consectetur nulla, a suscipit ipsum tortor vel magna. Nam sed posuere sem. Aliquam tincidunt in tortor ac cursus. Ut pellentesque urna in purus scelerisque, et imperdiet orci mollis. Fusce ultrices massa nisl, nec facilisis purus scelerisque ut. Pellentesque malesuada scelerisque diam, et dignissim orci ultricies vitae. Proin molestie ac augue at aliquet. Pellentesque fermentum dapibus purus ac porta. Nunc eu augue dui. Proin lacinia, enim at pretium vulputate, arcu arcu egestas ipsum, vel vehicula metus neque sed ex. Suspendisse vitae odio eu sapien pellentesque fermentum in nec leo. -Maecenas accumsan urna eros, id consequat felis maximus ac. Quisque id dictum sapien, et maximus justo. Sed blandit sit amet arcu ac vulputate. Sed bibendum maximus volutpat. Mauris dignissim purus vel leo ultrices aliquet. Aenean vitae sem rhoncus, vehicula sem quis, cursus metus. Duis hendrerit metus a fringilla sollicitudin. Morbi pellentesque velit eget posuere rhoncus. -Maecenas ullamcorper congue egestas. Aliquam maximus ullamcorper velit, vel ultrices ex pulvinar a. Sed convallis, metus eu facilisis lacinia, ex ante sollicitudin ipsum, vitae malesuada nibh nulla quis nulla. Aenean molestie, nulla commodo mattis viverra, purus mi ultrices ligula, ac accumsan libero velit sed est. Praesent cursus tincidunt ipsum nec sodales. Sed aliquet risus et odio lacinia, ac molestie libero posuere. Donec ultrices quam sapien, vel maximus eros malesuada eu. Integer tempor scelerisque laoreet. Vestibulum placerat rhoncus consequat. Nullam hendrerit, eros faucibus dictum faucibus, urna odio lobortis orci, ac suscipit ipsum lorem condimentum odio. Sed sollicitudin at odio sed viverra. Nulla aliquet facilisis ultrices. -Etiam vitae tempor tellus. Suspendisse potenti. Mauris sit amet odio diam. Vivamus at dignissim dui. Cras at nisl ut lorem condimentum vehicula. Cras vel nunc lectus. Morbi consequat dictum aliquet. Sed fermentum elementum ipsum, tempor fringilla libero varius et. -Mauris sed dictum purus, et ullamcorper tortor. Duis quis gravida urna. Integer pellentesque, ipsum at laoreet commodo, arcu dui semper lectus, sit amet consectetur diam nibh ut justo. Nam condimentum tincidunt arcu nec congue. Sed vulputate ligula at quam ultricies, et pretium justo consequat. Aenean nunc nisi, vehicula sed velit ut, pharetra egestas orci. Mauris pharetra erat tincidunt, gravida ipsum vitae, fermentum purus. Pellentesque sodales libero non purus ornare fermentum. Aliquam laoreet dignissim sem, et placerat turpis tincidunt non. -Cras bibendum ornare consequat. Donec non enim blandit, varius lorem sit amet, aliquam nibh. Morbi orci libero, egestas vitae arcu a, mollis interdum ante. Nullam vulputate risus enim, sit amet bibendum risus semper ut. Donec at orci sit amet ante hendrerit iaculis a non massa. Vestibulum nisi velit, dictum quis ullamcorper in, rhoncus sed ipsum. Curabitur sollicitudin diam dictum volutpat suscipit. Curabitur a elit et orci pellentesque venenatis ut rhoncus tellus. Proin nec metus non leo pharetra aliquet. Sed dolor augue, vehicula in odio vitae, cursus tincidunt eros. Praesent tempus luctus metus ac maximus. Phasellus non nibh quis eros rhoncus eleifend quis vitae nisi. -In ante enim, convallis in mollis nec, rutrum a nulla. Sed a urna ac nibh pharetra volutpat sit amet vel tortor. Quisque sodales tincidunt felis, et sollicitudin ligula venenatis vel. Pellentesque non justo eu diam sodales venenatis sed nec lacus. Nulla dignissim, sapien non pharetra scelerisque, risus massa dictum purus, congue convallis ante diam nec erat. Maecenas malesuada efficitur neque, a condimentum erat finibus vitae. Mauris quis dolor cursus nisi eleifend convallis sed id diam. Sed non pellentesque nisi. Praesent ut velit et tellus eleifend lacinia. Sed suscipit rutrum viverra. Pellentesque eget malesuada diam. Ut sed leo egestas, gravida odio aliquam, sodales nulla. In in lobortis nunc. Mauris et ex volutpat, elementum risus a, maximus magna. -Donec at odio sit amet nulla maximus semper at in elit. Vestibulum fringilla a tortor ut tristique. Proin a tristique mi. Cras libero ipsum, blandit ut volutpat non, convallis ac erat. Vivamus condimentum nunc dignissim, vestibulum tellus ut, tempus ante. Suspendisse fringilla nisi id arcu blandit pellentesque. Etiam vel elit velit. Donec ac sem consectetur, pharetra nisi a, facilisis turpis. Integer nec aliquam risus. Nullam posuere ligula eu purus rhoncus, nec fringilla nulla blandit. -Fusce condimentum sapien in elit dictum, ullamcorper vulputate ligula lacinia. Donec placerat elit dignissim mauris pellentesque malesuada. Maecenas semper lectus quis orci varius porta. Cras nec magna vitae ligula gravida bibendum. Pellentesque gravida purus sapien, eu egestas orci cursus sodales. Fusce varius ultrices enim quis faucibus. Aenean vel lobortis ex. Integer scelerisque nulla vel ligula pretium eleifend. Morbi lacus ligula, semper nec aliquet a, pulvinar nec ligula. Duis est orci, varius at ultricies ut, molestie id arcu. Nulla molestie lectus pharetra tortor cursus, eget gravida justo tempor. Fusce non ornare ante. In eu tortor venenatis, aliquet augue a, euismod purus. Duis et nibh venenatis, gravida nisi ut, sodales mi. Ut at ligula pretium, laoreet velit rutrum, sodales neque. Nulla quam nulla, mattis quis erat in, bibendum laoreet sem. -Donec posuere diam et est fringilla, sit amet malesuada lectus auctor. Nulla diam ante, tincidunt eu elementum in, congue vitae ante. Curabitur vulputate, nunc at posuere consectetur, nunc sem sagittis felis, non iaculis elit arcu eu metus. Pellentesque a eros dui. Phasellus commodo risus ante, at placerat augue ultricies a. Mauris in elementum velit. Curabitur quis odio vel tellus lacinia porttitor. Mauris nec mollis est, eget efficitur tellus. -In porttitor vel lectus vel finibus. Nunc quis quam ac nunc fringilla tempor. Pellentesque vel malesuada nunc. Suspendisse interdum pretium turpis, eget pretium tellus facilisis at. Etiam id ipsum eu velit elementum vehicula. Mauris vulputate lacus ac elit semper, a sodales metus varius. In semper ligula non erat vulputate, nec sollicitudin lectus gravida. Suspendisse porttitor sapien quis consectetur euismod. Maecenas consectetur scelerisque augue, eu iaculis dolor vulputate eget. In lobortis tristique mauris eu elementum. Proin lacus libero, suscipit id eleifend tincidunt, tincidunt vel felis. Aenean vulputate leo nec odio vehicula, venenatis tempus purus rutrum. -Curabitur non mauris vitae velit tempor dictum. Morbi id congue nunc. Aliquam imperdiet nec eros vitae pulvinar. Donec ut lobortis mauris. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Vivamus pellentesque ante sem, vitae feugiat ante lobortis ut. Aliquam felis est, accumsan eget lacus at, gravida gravida ipsum. Etiam congue est justo, sed elementum erat aliquet quis. Aenean porttitor massa pellentesque, efficitur lacus non, scelerisque justo. Integer commodo turpis nibh, eget tincidunt mauris consequat id. Sed convallis egestas rhoncus. In faucibus mi sit amet dolor tempor, non ullamcorper erat mollis. -Quisque tincidunt feugiat volutpat. Nunc consectetur sapien finibus ipsum tincidunt, sit amet facilisis justo suscipit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Pellentesque scelerisque massa eros, eget tempor lorem rutrum sit amet. Mauris faucibus libero at est sodales elementum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum dapibus ipsum ac quam placerat, eget lacinia lorem lobortis. Pellentesque eget velit nunc. Integer elementum justo in erat lobortis scelerisque. Sed ultrices dapibus quam quis ullamcorper. Duis auctor commodo placerat. Vivamus sed gravida turpis. Cras et nisl dignissim, pellentesque quam sed, eleifend nibh. Pellentesque et metus vel ipsum interdum bibendum. -Nam porttitor tristique metus eget elementum. Nullam eget risus commodo, malesuada dolor sit amet, tempus leo. Etiam eu iaculis neque, in tristique enim. Sed fringilla convallis leo id eleifend. Donec id purus suscipit, iaculis elit a, luctus felis. Praesent laoreet vulputate mi, eget cursus odio venenatis vitae. Curabitur egestas sem enim, non luctus erat tincidunt consectetur. Sed non viverra tellus. -Donec sodales ex eu neque condimentum feugiat. Integer elementum efficitur ligula quis aliquam. Quisque gravida tincidunt mauris. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Quisque ut vehicula tellus. Maecenas ultricies est ac quam aliquam, eget tincidunt eros bibendum. Curabitur a viverra risus, ut dignissim lacus. Praesent pharetra vulputate porttitor. Suspendisse cursus egestas nisl, in mattis risus imperdiet non. Ut tempus tortor malesuada dapibus tristique. Sed blandit ligula non lacus tincidunt pellentesque. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Fusce semper mattis ante, in efficitur ligula egestas sit amet. -Nullam eu dapibus nisi, nec fermentum enim. Duis orci odio, lobortis nec ullamcorper a, faucibus fringilla est. Aenean convallis, lorem non euismod venenatis, sapien ipsum pulvinar ipsum, non auctor arcu massa in nisi. Maecenas tincidunt sapien ante, in consequat dui accumsan eget. Phasellus id tellus ac libero iaculis fermentum. Vestibulum sagittis sapien quis porttitor facilisis. Cras euismod sollicitudin nisi, et pretium sapien hendrerit a. Ut auctor vitae neque id pellentesque. Donec at dui blandit, ultricies orci et, aliquet mauris. Praesent porttitor massa vel diam ultrices, ac convallis tellus tincidunt. Duis molestie condimentum nisl non venenatis. Pellentesque scelerisque odio ut lobortis porttitor. Donec congue interdum dolor, non sagittis tellus mattis vel. Fusce commodo augue fringilla semper gravida. -Sed placerat erat nulla, at commodo enim dapibus id. Integer augue dui, aliquet a eros at, euismod luctus erat. Donec vitae tempus enim. In luctus posuere sollicitudin. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Phasellus ultrices a felis tincidunt condimentum. Integer nulla augue, pellentesque eu facilisis id, congue dignissim ligula. Phasellus vestibulum blandit vulputate. Morbi a tempus eros. Maecenas ac metus vehicula massa egestas rhoncus. Nam accumsan porttitor rutrum. Morbi sollicitudin eros vestibulum tortor finibus, nec porta orci eleifend. Curabitur id suscipit quam. -Donec vehicula metus massa, accumsan vulputate dolor bibendum vel. Curabitur tincidunt magna sit amet massa commodo, quis porttitor libero sollicitudin. Aenean sollicitudin vehicula dignissim. Duis rhoncus varius leo, vitae eleifend eros tristique quis. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Quisque fermentum nulla purus, nec aliquam nunc ultrices ut. Praesent vestibulum risus eu dolor ultricies eleifend. Nulla vitae hendrerit lorem, a pellentesque eros. Praesent pellentesque sit amet arcu ac hendrerit. -Ut scelerisque, velit eu rhoncus eleifend, dolor dui ultricies dolor, id fermentum lectus nunc nec sem. Donec eget nisi eget augue consequat pretium. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed sollicitudin eget magna sed varius. Nam ultricies, eros non dapibus varius, eros elit finibus metus, quis semper lacus lectus in turpis. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Praesent ornare, sapien vel gravida semper, risus risus luctus ligula, in vestibulum nibh arcu ac tortor. In cursus tempus justo eget mollis. Phasellus eget mauris hendrerit, venenatis dolor quis, euismod neque. Suspendisse potenti. -Suspendisse lorem lectus, accumsan eget ipsum ac, commodo luctus velit. Vivamus non ligula enim. In hendrerit turpis ut mauris commodo sagittis. Sed dapibus dolor vel turpis malesuada, id tincidunt ipsum convallis. Nulla in convallis enim, at dignissim mi. Etiam at mauris eget eros viverra malesuada. Etiam malesuada ligula eu libero condimentum pretium. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Sed eget quam non purus malesuada tristique. -Aenean at arcu felis. Donec consectetur ante a tellus pellentesque sollicitudin. Duis ut arcu sapien. Aenean quis lacus a turpis porttitor egestas ut nec nisi. Aenean viverra nec nunc quis lacinia. Nulla dapibus eros ac placerat blandit. Morbi tincidunt porttitor sapien, eu aliquet ipsum ullamcorper at. Phasellus eleifend lacus et mauris mollis gravida. Nullam vitae luctus mauris. Integer malesuada nec ligula non gravida. Nullam at facilisis neque. Vestibulum ac est a libero sagittis tempor a et urna. Suspendisse ac sollicitudin leo, in aliquet augue. Curabitur non vulputate mi. Ut eget euismod ex, ut placerat leo. Praesent suscipit lectus magna, ac bibendum risus dapibus id. -Cras consequat vel nunc id porttitor. Suspendisse tristique nunc leo, quis fermentum diam ultrices ac. Donec scelerisque mollis porttitor. Maecenas interdum dui orci, ut volutpat purus iaculis ac. Proin sed odio est. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Vestibulum malesuada turpis nunc, nec tincidunt risus ultricies eu. In pellentesque a leo quis aliquam. Suspendisse potenti. Cras volutpat nibh arcu, sit amet molestie purus rhoncus et. -Phasellus mi neque, placerat sed justo laoreet, aliquet sodales sem. Mauris vulputate aliquam sem, ac consectetur dolor iaculis eget. Nunc lobortis sapien lectus, ac consectetur nisi porta ac. In in sem nec massa consequat viverra nec aliquam urna. Proin maximus posuere tellus quis imperdiet. Suspendisse lobortis viverra aliquet. Duis laoreet, sem at ullamcorper consequat, eros orci maximus urna, non laoreet sapien orci ac ante. Quisque vulputate, magna ac venenatis tincidunt, nunc nibh convallis elit, sit amet mattis quam ipsum condimentum sem. Nunc maximus, nibh in lobortis dapibus, est erat lobortis tortor, eu euismod mi leo eu tellus. Praesent sit amet orci at quam porttitor cursus nec vel mauris. Nullam at sagittis tortor. Vivamus fermentum, quam nec egestas lobortis, velit velit molestie quam, in finibus libero mauris a lacus. -Sed nec dolor a lorem semper rutrum nec sit amet est. Vestibulum blandit bibendum nibh at commodo. Aenean sit amet purus dolor. Curabitur consequat volutpat aliquam. Nullam erat lorem, condimentum quis ultrices at, imperdiet at est. Nullam placerat mi sed facilisis rutrum. Proin metus felis, vulputate lacinia urna sit amet, rutrum fringilla felis. Donec quam lectus, porta sed elit et, dignissim luctus libero. -Suspendisse a augue ante. Phasellus hendrerit elit ut maximus consectetur. Aliquam id nisl ac lectus sodales rutrum quis vehicula purus. Vivamus sollicitudin turpis ac felis ullamcorper, id dictum ante iaculis. Fusce a libero est. In bibendum varius enim, gravida ultricies leo placerat et. Nunc ante erat, hendrerit sed sapien ac, molestie semper libero. Sed sagittis, quam non congue imperdiet, tellus leo ullamcorper metus, eget vestibulum tortor dui at dolor. Mauris est sem, tincidunt vel luctus non, tincidunt eu ipsum. Nullam a feugiat dui. Nunc vel velit vel ipsum sollicitudin dictum. Ut cursus imperdiet nibh, et aliquam ligula lacinia ac. -Cras erat felis, commodo ut posuere id, vehicula id neque. Vestibulum urna dolor, gravida vestibulum libero sed, pretium molestie mauris. Nullam in efficitur ipsum. Donec et ipsum finibus, bibendum diam vel, placerat est. Donec vitae gravida felis. Etiam augue justo, finibus id eros nec, consectetur lobortis risus. Aenean varius accumsan quam, ac pulvinar nunc pretium at. Mauris vitae risus maximus, eleifend leo ac, maximus nunc. Cras nibh ante, rhoncus eu eleifend eget, accumsan eget ex. -Vivamus tincidunt fermentum aliquet. Etiam et venenatis diam. Sed scelerisque convallis eros non pulvinar. Ut dignissim justo eros, id rhoncus magna mollis quis. Nullam lobortis odio at placerat feugiat. Morbi feugiat est eu mauris pellentesque efficitur. Praesent posuere arcu urna, eu cursus leo fermentum sit amet. Curabitur sit amet aliquam odio. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Nunc malesuada nulla est, et varius nulla condimentum sed. Sed sit amet eros felis. Nullam vitae pretium lectus. -Etiam at massa vel urna varius accumsan. Proin non porttitor purus, nec facilisis libero. Quisque tristique, mi ac tristique vulputate, ipsum nibh porta urna, et tempus ex orci eu libero. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Proin non pellentesque elit, at vehicula justo. Ut facilisis mi sit amet est faucibus, id lobortis est varius. Nulla placerat justo lorem, quis condimentum nulla dapibus quis. Curabitur blandit non lorem at ultrices. Duis nunc mauris, cursus porttitor mauris mattis, consequat luctus neque. -Duis congue non quam eget dignissim. Etiam molestie metus et arcu sollicitudin cursus. Suspendisse cursus luctus rhoncus. Proin lorem metus, bibendum pharetra auctor vitae, malesuada eget risus. Aenean ac libero quis metus lacinia dignissim nec vitae tellus. Nulla consectetur semper ipsum, sed imperdiet elit lobortis sit amet. Vivamus iaculis arcu ut mattis iaculis. In metus tortor, euismod sit amet ligula eget, tincidunt euismod dolor. Aliquam eget elit at est faucibus aliquet sit amet quis leo. -Praesent lectus dolor, tincidunt et urna eget, consectetur volutpat libero. Quisque placerat vel sem ac accumsan. Nulla facilisi. Praesent facilisis condimentum quam. Vivamus malesuada nibh ut dignissim commodo. Morbi a egestas leo. Sed viverra, purus eu mattis varius, nibh nunc convallis est, et tristique nulla eros ut augue. Curabitur a dui tortor. Pellentesque viverra nisi ut risus laoreet vehicula. -Suspendisse potenti. Quisque et condimentum tellus. Vestibulum mollis sollicitudin risus, ac suscipit ex interdum ac. Vivamus mattis velit non lectus consectetur aliquam. Maecenas commodo iaculis eros ac suscipit. Quisque tempor libero in ultricies luctus. Phasellus dictum, orci in suscipit fringilla, est dui imperdiet purus, vitae tempor tellus odio et est. Vestibulum iaculis id felis in facilisis. Morbi luctus tortor sit amet dui varius iaculis. Sed gravida ipsum lectus, id tincidunt dui dapibus eget. Proin at vestibulum libero, nec ornare sapien. Quisque odio mi, maximus in pretium et, hendrerit at odio. -Nunc ullamcorper mattis laoreet. In dolor justo, imperdiet eget libero id, imperdiet laoreet erat. Donec molestie dictum leo, nec luctus ipsum viverra nec. Maecenas scelerisque metus sed vulputate ullamcorper. Maecenas varius eget felis vel porttitor. Sed sem diam, pellentesque vitae egestas ut, varius at urna. Donec eu metus commodo augue porta eleifend. Maecenas id porta erat. Aenean sed nunc orci. Ut nec tristique ipsum, ut egestas purus. Nullam egestas dictum tristique. Sed lacinia mauris quis cursus tristique. Quisque ac urna id diam elementum vulputate eu eu purus. -Aliquam sit amet purus bibendum, placerat massa quis, malesuada nisl. Nullam orci justo, egestas eget neque maximus, cursus porttitor velit. Donec vel sodales risus. Curabitur lorem dui, finibus id est a, hendrerit luctus velit. Praesent aliquet molestie felis, nec dictum leo tincidunt vel. Cras rutrum tempus egestas. Sed in ex interdum, lobortis nunc a, pharetra diam. Ut sed risus scelerisque purus rutrum rhoncus ut eget urna. Praesent mollis arcu vel auctor luctus. Nullam risus magna, iaculis non condimentum id, maximus sed diam. Integer porttitor ornare metus sed mattis. Nulla facilisi. Proin facilisis vestibulum dolor, sed hendrerit metus convallis a. Cras rutrum est eget mauris laoreet, nec blandit sapien euismod. -Nulla egestas auctor arcu in porttitor. Integer tincidunt ex non laoreet tincidunt. Suspendisse vitae urna ut nibh auctor viverra nec in lacus. In finibus sagittis velit, non rhoncus sapien tristique non. Suspendisse eu urna cursus, varius turpis vel, semper est. Donec orci augue, mollis sed auctor in, eleifend vitae justo. Cras commodo orci interdum turpis suscipit laoreet. Quisque venenatis lacus ut leo molestie pellentesque. Nunc id congue risus. Donec sagittis erat nunc, non consectetur erat euismod eu. Nam at velit nisl. \ No newline at end of file diff --git a/nitrite/build.gradle b/nitrite/build.gradle index 4904c276d..a99e9ba56 100644 --- a/nitrite/build.gradle +++ b/nitrite/build.gradle @@ -44,27 +44,28 @@ repositories { dependencies { api platform(project(':nitrite-bom')) api "org.slf4j:slf4j-api" - api "org.objenesis:objenesis" api "org.jasypt:jasypt" - annotationProcessor "org.projectlombok:lombok:1.18.22" + annotationProcessor "org.projectlombok:lombok:1.18.24" - testAnnotationProcessor "org.projectlombok:lombok:1.18.20" + testAnnotationProcessor "org.projectlombok:lombok:1.18.24" testImplementation "junit:junit:4.13.2" - testImplementation "org.mockito:mockito-core:4.1.0" - testImplementation "uk.co.jemos.podam:podam:7.2.7.RELEASE" - testImplementation "com.github.javafaker:javafaker:1.0.2" - testImplementation "org.apache.lucene:lucene-core:8.11.0" - testImplementation "org.apache.lucene:lucene-analyzers-common:8.11.0" - testImplementation "org.apache.lucene:lucene-queryparser:8.11.0" - testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.14.1" - testImplementation "org.apache.logging.log4j:log4j-core:2.15.0" - testImplementation "org.awaitility:awaitility:4.1.1" - testImplementation "joda-time:joda-time:2.10.13" + testImplementation "org.mockito:mockito-core:4.6.1" + testImplementation 'uk.co.jemos.podam:podam:7.2.9.RELEASE' + testImplementation ("com.github.javafaker:javafaker:1.0.2") { + exclude module: 'snakeyaml' + } + testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2" + testImplementation "org.apache.logging.log4j:log4j-core:2.17.2" + testImplementation "org.awaitility:awaitility:4.2.0" + testImplementation 'joda-time:joda-time:2.10.14' testImplementation "org.meanbean:meanbean:2.0.3" - testImplementation "com.fasterxml.jackson.core:jackson-databind:2.13.0" + testImplementation ("com.fasterxml.jackson.core:jackson-databind:2.13.3") { + exclude module: 'org.yaml' + } testImplementation "commons-io:commons-io:2.11.0" - testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:3.0.1" - testImplementation "com.sun.xml.bind:jaxb-impl:3.0.2" + testImplementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0' + testImplementation 'com.sun.xml.bind:jaxb-impl:4.0.0' + testImplementation 'org.yaml:snakeyaml:1.30' } test { diff --git a/nitrite/src/main/java/org/dizitart/no2/Nitrite.java b/nitrite/src/main/java/org/dizitart/no2/Nitrite.java index d012d5bc4..cd5a39fa1 100644 --- a/nitrite/src/main/java/org/dizitart/no2/Nitrite.java +++ b/nitrite/src/main/java/org/dizitart/no2/Nitrite.java @@ -18,17 +18,21 @@ import org.dizitart.no2.collection.NitriteCollection; import org.dizitart.no2.common.Constants; +import org.dizitart.no2.common.util.ObjectUtils; import org.dizitart.no2.exceptions.NitriteIOException; import org.dizitart.no2.exceptions.ValidationException; import org.dizitart.no2.repository.ObjectRepository; -import org.dizitart.no2.store.StoreMetaData; +import org.dizitart.no2.repository.EntityDecorator; import org.dizitart.no2.store.NitriteStore; +import org.dizitart.no2.store.StoreMetaData; import org.dizitart.no2.transaction.Session; import java.util.Map; import java.util.Set; import static org.dizitart.no2.common.Constants.RESERVED_NAMES; +import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryName; +import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryNameByDecorator; import static org.dizitart.no2.common.util.ValidationUtils.notEmpty; import static org.dizitart.no2.common.util.ValidationUtils.notNull; @@ -112,6 +116,33 @@ static NitriteBuilder builder() { */ ObjectRepository getRepository(Class type, String key); + /** + * Opens a type-safe object repository using a {@link EntityDecorator}. If the repository + * does not exist it will be created automatically and returned. If a + * repository is already opened, it is returned as is. + *

+ * The returned repository is thread-safe for concurrent use. + * + * @param the type parameter + * @param entityDecorator the entityDecorator + * @return the repository + */ + ObjectRepository getRepository(EntityDecorator entityDecorator); + + /** + * Opens a type-safe object repository using a {@link EntityDecorator} and a key identifier + * from the store. If the repository does not exist it will be created automatically and + * returned. If a repository is already opened, it is returned as is. + *

+ * The returned repository is thread-safe for concurrent use. + * + * @param the type parameter + * @param entityDecorator the entityDecorator + * @param key the key + * @return the repository + */ + ObjectRepository getRepository(EntityDecorator entityDecorator, String key); + /** * Destroys a {@link NitriteCollection} without opening it first. * @@ -128,7 +159,7 @@ static NitriteBuilder builder() { void destroyRepository(Class type); /** - * Destroys an keyed-{@link ObjectRepository} without opening it first. + * Destroys a keyed-{@link ObjectRepository} without opening it first. * * @param the type parameter * @param type the type @@ -136,6 +167,23 @@ static NitriteBuilder builder() { */ void destroyRepository(Class type, String key); + /** + * Destroys an {@link ObjectRepository} without opening it first. + * + * @param the type parameter + * @param type the type + */ + void destroyRepository(EntityDecorator type); + + /** + * Destroys a keyed-{@link ObjectRepository} without opening it first. + * + * @param the type parameter + * @param type the type + * @param key the key + */ + void destroyRepository(EntityDecorator type, String key); + /** * Gets the set of all {@link NitriteCollection}s' names saved in the store. * @@ -157,7 +205,7 @@ static NitriteBuilder builder() { * * @return the set of all registered classes' names. */ - Map> listKeyedRepository(); + Map> listKeyedRepositories(); /** * Checks whether the store has any unsaved changes. @@ -226,7 +274,8 @@ default boolean hasCollection(String name) { */ default boolean hasRepository(Class type) { checkOpened(); - return listRepositories().contains(type.getName()); + String name = findRepositoryName(type, null); + return listRepositories().contains(name); } /** @@ -239,8 +288,36 @@ default boolean hasRepository(Class type) { */ default boolean hasRepository(Class type, String key) { checkOpened(); - return listKeyedRepository().containsKey(key) - && listKeyedRepository().get(key).contains(type.getName()); + String entityName = ObjectUtils.getEntityName(type); + return listKeyedRepositories().containsKey(key) + && listKeyedRepositories().get(key).contains(entityName); + } + + /** + * Checks whether a particular {@link ObjectRepository} exists in the store. + * + * @param the type parameter + * @param entityDecorator entityDecorator + * @return true if the repository exists; otherwise false. + */ + default boolean hasRepository(EntityDecorator entityDecorator) { + checkOpened(); + String name = findRepositoryNameByDecorator(entityDecorator, null); + return listRepositories().contains(name); + } + + /** + * Checks whether a particular keyed-{@link ObjectRepository} exists in the store. + * + * @param the type parameter. + * @param entityDecorator entityDecorator. + * @param key the key, which will be appended to the repositories name. + * @return true if the repository exists; otherwise false. + */ + default boolean hasRepository(EntityDecorator entityDecorator, String key) { + checkOpened(); + return listKeyedRepositories().containsKey(key) + && listKeyedRepositories().get(key).contains(entityDecorator.getEntityName()); } /** @@ -254,7 +331,7 @@ default void validateCollectionName(String name) { for (String reservedName : RESERVED_NAMES) { if (name.contains(reservedName)) { - throw new ValidationException("name cannot contain " + reservedName); + throw new ValidationException("Name cannot contain " + reservedName); } } } @@ -264,7 +341,7 @@ default void validateCollectionName(String name) { */ default void checkOpened() { if (getStore() == null || getStore().isClosed()) { - throw new NitriteIOException("store is closed"); + throw new NitriteIOException("Store is closed"); } } } diff --git a/nitrite/src/main/java/org/dizitart/no2/NitriteBuilder.java b/nitrite/src/main/java/org/dizitart/no2/NitriteBuilder.java index 3fa969a28..8aafb1913 100644 --- a/nitrite/src/main/java/org/dizitart/no2/NitriteBuilder.java +++ b/nitrite/src/main/java/org/dizitart/no2/NitriteBuilder.java @@ -85,19 +85,15 @@ public NitriteBuilder addMigrations(Migration... migrations) { * @return the nitrite builder */ public NitriteBuilder schemaVersion(Integer version) { - this.nitriteConfig.schemaVersion(version); + this.nitriteConfig.currentSchemaVersion(version); return this; } /** - * Opens or creates a new nitrite database backed by mvstore. If it is an in-memory store, + * Opens or creates a new nitrite database. If it is an in-memory store, * then it will create a new one. If it is a file based store, and if the file does not - * exists, then it will create a new file store and open; otherwise it will + * exist, then it will create a new file store and open; otherwise it will * open the existing file store. - *

- * NOTE: If the database is corrupted somehow then at the time of opening, it will - * try to repair it using the last known good version. If still it fails to - * recover, then it will throw a {@link org.dizitart.no2.exceptions.NitriteIOException}. * * @return the nitrite database instance. * @throws org.dizitart.no2.exceptions.NitriteIOException if unable to create a new in-memory database. @@ -106,35 +102,36 @@ public NitriteBuilder schemaVersion(Integer version) { */ public Nitrite openOrCreate() { this.nitriteConfig.autoConfigure(); - return new NitriteDatabase(nitriteConfig); + Runtime.getRuntime().addShutdownHook(new Thread(ThreadPoolManager::shutdownAllThreadPools)); + NitriteDatabase db = new NitriteDatabase(nitriteConfig); + db.initialize(null, null); + return db; } /** - * Opens or creates a new nitrite database backed by mvstore. If it is an in-memory store, + * Opens or creates a new nitrite database. If it is an in-memory store, * then it will create a new one. If it is a file based store, and if the file does not - * exists, then it will create a new file store and open; otherwise it will + * exist, then it will create a new file store and open; otherwise it will * open the existing file store. *

* While creating a new database, it will use the specified user credentials. * While opening an existing database, it will use the specified credentials * to open it. *

- *

- * NOTE: If the database is corrupted somehow then at the time of opening, it will - * try to repair it using the last known good version. If still it fails to - * recover, then it will throw a {@link org.dizitart.no2.exceptions.NitriteIOException}. * * @param username the username * @param password the password * @return the nitrite database instance. - * @throws NitriteSecurityException if the user credentials are wrong or one of them is empty string. + * @throws NitriteSecurityException if the user credentials are wrong or one of them is empty string. * @throws org.dizitart.no2.exceptions.NitriteIOException if unable to create a new in-memory database. * @throws org.dizitart.no2.exceptions.NitriteIOException if the database is corrupt and recovery fails. * @throws org.dizitart.no2.exceptions.NitriteIOException if the directory does not exist. */ public Nitrite openOrCreate(String username, String password) { this.nitriteConfig.autoConfigure(); - Runtime.getRuntime().addShutdownHook(new Thread(ThreadPoolManager::shutdownThreadPools)); - return new NitriteDatabase(username, password, nitriteConfig); + Runtime.getRuntime().addShutdownHook(new Thread(ThreadPoolManager::shutdownAllThreadPools)); + NitriteDatabase db = new NitriteDatabase(nitriteConfig); + db.initialize(username, password); + return db; } } diff --git a/nitrite/src/main/java/org/dizitart/no2/NitriteConfig.java b/nitrite/src/main/java/org/dizitart/no2/NitriteConfig.java index a432ae80e..77ed5bfca 100644 --- a/nitrite/src/main/java/org/dizitart/no2/NitriteConfig.java +++ b/nitrite/src/main/java/org/dizitart/no2/NitriteConfig.java @@ -80,7 +80,7 @@ public NitriteConfig() { */ public void fieldSeparator(String separator) { if (configured) { - throw new InvalidOperationException("cannot change the separator after database" + + throw new InvalidOperationException("Cannot change the separator after database" + " initialization"); } NitriteConfig.fieldSeparator = separator; @@ -94,7 +94,7 @@ public void fieldSeparator(String separator) { */ public NitriteConfig loadModule(NitriteModule module) { if (configured) { - throw new InvalidOperationException("cannot load module after database" + + throw new InvalidOperationException("Cannot load module after database" + " initialization"); } pluginManager.loadModule(module); @@ -110,13 +110,13 @@ public NitriteConfig loadModule(NitriteModule module) { @SuppressWarnings("Java8MapApi") public NitriteConfig addMigration(Migration migration) { if (configured) { - throw new InvalidOperationException("cannot add migration steps after database" + + throw new InvalidOperationException("Cannot add migration steps after database" + " initialization"); } if (migration != null) { - final int start = migration.getStartVersion(); - final int end = migration.getEndVersion(); + final int start = migration.getFromVersion(); + final int end = migration.getToVersion(); TreeMap targetMap = migrations.get(start); if (targetMap == null) { targetMap = new TreeMap<>(); @@ -137,9 +137,9 @@ public NitriteConfig addMigration(Migration migration) { * @param version the version * @return the nitrite config */ - public NitriteConfig schemaVersion(Integer version) { + public NitriteConfig currentSchemaVersion(Integer version) { if (configured) { - throw new InvalidOperationException("cannot add schema version info after database" + + throw new InvalidOperationException("Cannot add schema version info after database" + " initialization"); } this.schemaVersion = version; @@ -147,19 +147,19 @@ public NitriteConfig schemaVersion(Integer version) { } /** - * Auto configures nitrite database with default configuration values and + * Autoconfigures nitrite database with default configuration values and * default built-in plugins. */ public void autoConfigure() { if (configured) { - throw new InvalidOperationException("cannot execute autoconfigure after database" + + throw new InvalidOperationException("Cannot execute autoconfigure after database" + " initialization"); } pluginManager.findAndLoadPlugins(); } /** - * Finds an {@link NitriteIndexer} by indexType. + * Finds a {@link NitriteIndexer} by indexType. * * @param indexType the type of {@link NitriteIndexer} to find. * @return the {@link NitriteIndexer} @@ -170,7 +170,7 @@ public NitriteIndexer findIndexer(String indexType) { nitriteIndexer.initialize(this); return nitriteIndexer; } else { - throw new IndexingException("no indexer found for index type " + indexType); + throw new IndexingException("No indexer found for index type " + indexType); } } diff --git a/nitrite/src/main/java/org/dizitart/no2/NitriteDatabase.java b/nitrite/src/main/java/org/dizitart/no2/NitriteDatabase.java index f355623ac..e0f6acfdb 100644 --- a/nitrite/src/main/java/org/dizitart/no2/NitriteDatabase.java +++ b/nitrite/src/main/java/org/dizitart/no2/NitriteDatabase.java @@ -27,19 +27,20 @@ import org.dizitart.no2.migration.MigrationManager; import org.dizitart.no2.repository.ObjectRepository; import org.dizitart.no2.repository.RepositoryFactory; -import org.dizitart.no2.store.StoreMetaData; +import org.dizitart.no2.repository.EntityDecorator; import org.dizitart.no2.store.NitriteMap; import org.dizitart.no2.store.NitriteStore; +import org.dizitart.no2.store.StoreMetaData; import org.dizitart.no2.store.UserAuthenticationService; import org.dizitart.no2.transaction.Session; -import java.io.File; import java.util.Map; import java.util.Set; import static org.dizitart.no2.common.Constants.NITRITE_VERSION; import static org.dizitart.no2.common.Constants.STORE_INFO; import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryName; +import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryNameByDecorator; import static org.dizitart.no2.common.util.StringUtils.isNullOrEmpty; /** @@ -60,16 +61,6 @@ class NitriteDatabase implements Nitrite { this.lockService = new LockService(); this.collectionFactory = new CollectionFactory(lockService); this.repositoryFactory = new RepositoryFactory(collectionFactory); - this.initialize(null, null); - } - - NitriteDatabase(String username, String password, NitriteConfig config) { - validateUserCredentials(username, password); - this.nitriteConfig = config; - this.lockService = new LockService(); - this.collectionFactory = new CollectionFactory(lockService); - this.repositoryFactory = new RepositoryFactory(collectionFactory); - this.initialize(username, password); } @Override @@ -91,6 +82,18 @@ public ObjectRepository getRepository(Class type, String key) { return repositoryFactory.getRepository(nitriteConfig, type, key); } + @Override + public ObjectRepository getRepository(EntityDecorator entityDecorator) { + checkOpened(); + return repositoryFactory.getRepository(nitriteConfig, entityDecorator); + } + + @Override + public ObjectRepository getRepository(EntityDecorator entityDecorator, String key) { + checkOpened(); + return repositoryFactory.getRepository(nitriteConfig, entityDecorator, key); + } + @Override public void destroyCollection(String name) { checkOpened(); @@ -111,6 +114,20 @@ public void destroyRepository(Class type, String key) { store.removeMap(mapName); } + @Override + public void destroyRepository(EntityDecorator type) { + checkOpened(); + String mapName = findRepositoryNameByDecorator(type, null); + store.removeMap(mapName); + } + + @Override + public void destroyRepository(EntityDecorator type, String key) { + checkOpened(); + String mapName = findRepositoryNameByDecorator(type, key); + store.removeMap(mapName); + } + @Override public Set listCollectionNames() { checkOpened(); @@ -124,7 +141,7 @@ public Set listRepositories() { } @Override - public Map> listKeyedRepository() { + public Map> listKeyedRepositories() { checkOpened(); return store.getKeyedRepositoryRegistry(); } @@ -165,7 +182,7 @@ public synchronized void close() { storeInfo.close(); if (nitriteConfig != null) { - // close all plugins + // close all plugins and store nitriteConfig.close(); } @@ -173,7 +190,7 @@ public synchronized void close() { } catch (NitriteIOException e) { throw e; } catch (Throwable error) { - throw new NitriteIOException("error while shutting down nitrite", error); + throw new NitriteIOException("Error occurred while closing the database", error); } } @@ -184,9 +201,9 @@ public void commit() { try { store.commit(); } catch (Exception e) { - throw new NitriteIOException("failed to commit changes", e); + throw new NitriteIOException("Error occurred while committing the database", e); } - log.debug("Unsaved changes committed successfully."); + log.debug("Unsaved changes has been committed successfully."); } } @@ -205,20 +222,11 @@ public Session createSession() { return new Session(this, lockService); } - private void validateUserCredentials(String username, String password) { - if (isNullOrEmpty(username)) { - throw new NitriteSecurityException("username cannot be empty"); - } - if (isNullOrEmpty(password)) { - throw new NitriteSecurityException("password cannot be empty"); - } - } - - private void initialize(String username, String password) { + public void initialize(String username, String password) { + validateUserCredentials(username, password); try { nitriteConfig.initialize(); store = nitriteConfig.getNitriteStore(); - boolean isExisting = isExisting(); store.openOrCreate(); prepareDatabaseMetaData(); @@ -227,29 +235,35 @@ private void initialize(String username, String password) { migrationManager.doMigrate(); UserAuthenticationService userAuthenticationService = new UserAuthenticationService(store); - userAuthenticationService.authenticate(username, password, isExisting); - } catch (NitriteException e) { - log.error("Error while initializing the database", e); - if (store != null && !store.isClosed()) { - try { - store.close(); - } catch (Exception ex) { - log.error("Error while closing the database", ex); - throw new NitriteIOException("failed to close database", ex); - } - } - throw e; + userAuthenticationService.authenticate(username, password); } catch (Exception e) { - log.error("Error while initializing the database", e); + log.error("Error occurred while initializing the database", e); if (store != null && !store.isClosed()) { try { store.close(); } catch (Exception ex) { - log.error("Error while closing the database"); - throw new NitriteIOException("failed to close database", ex); + log.error("Error occurred while closing the database"); + throw new NitriteIOException("Failed to close database", ex); } } - throw new NitriteIOException("failed to initialize database", e); + if (e instanceof NitriteException) { + throw e; + } else { + throw new NitriteIOException("Failed to initialize database", e); + } + } + } + + private void validateUserCredentials(String username, String password) { + if (isNullOrEmpty(username) && isNullOrEmpty(password)) { + return; + } + + if (isNullOrEmpty(username)) { + throw new NitriteSecurityException("Username is required"); + } + if (isNullOrEmpty(password)) { + throw new NitriteSecurityException("Password is required"); } } @@ -266,13 +280,4 @@ private void prepareDatabaseMetaData() { storeInfo.put(STORE_INFO, storeMetadata.getInfo()); } } - - private boolean isExisting() { - String filePath = store.getStoreConfig().filePath(); - if (!isNullOrEmpty(filePath)) { - File dbFile = new File(filePath); - return dbFile.exists(); - } - return false; - } } diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/CollectionFactory.java b/nitrite/src/main/java/org/dizitart/no2/collection/CollectionFactory.java index 52a7c8edc..ea2973230 100644 --- a/nitrite/src/main/java/org/dizitart/no2/collection/CollectionFactory.java +++ b/nitrite/src/main/java/org/dizitart/no2/collection/CollectionFactory.java @@ -56,12 +56,12 @@ public CollectionFactory(LockService lockService) { * * @param name the name * @param nitriteConfig the nitrite config - * @param writeCatalogue the write catalogue + * @param writeCatalogue to write catalogue * @return the collection */ public NitriteCollection getCollection(String name, NitriteConfig nitriteConfig, boolean writeCatalogue) { - notNull(nitriteConfig, "configuration is null while creating collection"); - notEmpty(name, "collection name is null or empty"); + notNull(nitriteConfig, "Configuration is null while creating collection"); + notEmpty(name, "Collection name is null or empty"); Lock lock = lockService.getWriteLock(this.getClass().getName()); try { @@ -83,25 +83,24 @@ public NitriteCollection getCollection(String name, NitriteConfig nitriteConfig, private NitriteCollection createCollection(String name, NitriteConfig nitriteConfig, boolean writeCatalog) { NitriteStore store = nitriteConfig.getNitriteStore(); - NitriteMap nitriteMap = store.openMap(name, NitriteId.class, Document.class); - NitriteCollection collection = new DefaultNitriteCollection(name, nitriteMap, nitriteConfig, lockService); if (writeCatalog) { // ignore repository request if (store.getRepositoryRegistry().contains(name)) { - nitriteMap.close(); - collection.close(); - throw new ValidationException("a repository with same name already exists"); + throw new ValidationException("A repository with same name already exists"); } for (Set set : store.getKeyedRepositoryRegistry().values()) { if (set.contains(name)) { - nitriteMap.close(); - collection.close(); - throw new ValidationException("a keyed repository with same name already exists"); + throw new ValidationException("A keyed repository with same name already exists"); } } + } + + NitriteMap nitriteMap = store.openMap(name, NitriteId.class, Document.class); + NitriteCollection collection = new DefaultNitriteCollection(name, nitriteMap, nitriteConfig, lockService); + if (writeCatalog) { collectionMap.put(name, collection); StoreCatalog storeCatalog = store.getCatalog(); storeCatalog.writeCollectionEntry(name); @@ -118,11 +117,13 @@ public void clear() { try { lock.lock(); for (NitriteCollection collection : collectionMap.values()) { - collection.close(); + if (collection.isOpen()) { + collection.close(); + } } collectionMap.clear(); } catch (Exception e) { - throw new NitriteIOException("failed to close a collection", e); + throw new NitriteIOException("Failed to close a collection", e); } finally { lock.unlock(); } diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/DefaultNitriteCollection.java b/nitrite/src/main/java/org/dizitart/no2/collection/DefaultNitriteCollection.java index 403feae0e..aedb15e49 100644 --- a/nitrite/src/main/java/org/dizitart/no2/collection/DefaultNitriteCollection.java +++ b/nitrite/src/main/java/org/dizitart/no2/collection/DefaultNitriteCollection.java @@ -16,17 +16,17 @@ package org.dizitart.no2.collection; -import lombok.Getter; import org.dizitart.no2.NitriteConfig; import org.dizitart.no2.collection.events.CollectionEventInfo; import org.dizitart.no2.collection.events.CollectionEventListener; -import org.dizitart.no2.collection.meta.Attributes; import org.dizitart.no2.collection.operation.CollectionOperations; import org.dizitart.no2.common.Fields; import org.dizitart.no2.common.WriteResult; import org.dizitart.no2.common.concurrent.LockService; import org.dizitart.no2.common.event.EventBus; import org.dizitart.no2.common.event.NitriteEventBus; +import org.dizitart.no2.common.meta.Attributes; +import org.dizitart.no2.common.processors.Processor; import org.dizitart.no2.exceptions.IndexingException; import org.dizitart.no2.exceptions.InvalidOperationException; import org.dizitart.no2.exceptions.NitriteIOException; @@ -35,7 +35,6 @@ import org.dizitart.no2.index.IndexDescriptor; import org.dizitart.no2.index.IndexOptions; import org.dizitart.no2.index.IndexType; -import org.dizitart.no2.common.processors.Processor; import org.dizitart.no2.store.NitriteMap; import org.dizitart.no2.store.NitriteStore; @@ -63,8 +62,6 @@ class DefaultNitriteCollection implements NitriteCollection { private Lock readLock; private CollectionOperations collectionOperations; private EventBus, CollectionEventListener> eventBus; - - @Getter private volatile boolean isDropped; DefaultNitriteCollection(String name, NitriteMap nitriteMap, @@ -90,19 +87,6 @@ public void addProcessor(Processor processor) { } } - @Override - public void removeProcessor(Processor processor) { - notNull(processor, "a null processor cannot be removed"); - - try { - writeLock.lock(); - checkOpened(); - collectionOperations.removeProcessor(processor); - } finally { - writeLock.unlock(); - } - } - public WriteResult insert(Document[] documents) { notNull(documents, "a null document cannot be inserted"); containsNull(documents, "a null document cannot be inserted"); @@ -125,7 +109,7 @@ public WriteResult update(Document document, boolean insertIfAbsent) { if (document.hasId()) { return update(createUniqueFilter(document), document, updateOptions(false)); } else { - throw new NotIdentifiableException("update operation failed as no id value found for the document"); + throw new NotIdentifiableException("Update operation failed as the document does not have id"); } } } @@ -155,13 +139,13 @@ public WriteResult remove(Document document) { writeLock.unlock(); } } else { - throw new NotIdentifiableException("remove operation failed as no id value found for the document"); + throw new NotIdentifiableException("Document has no id, cannot remove by document"); } } public WriteResult remove(Filter filter, boolean justOne) { if ((filter == null || filter == Filter.ALL) && justOne) { - throw new InvalidOperationException("remove all cannot be combined with just once"); + throw new InvalidOperationException("Cannot remove all documents with justOne set to true"); } try { @@ -177,7 +161,7 @@ public void clear() { try { writeLock.lock(); checkOpened(); - nitriteMap.clear(); + collectionOperations.clear(); } finally { writeLock.unlock(); } @@ -316,9 +300,6 @@ public void drop() { checkOpened(); if (collectionOperations != null) { - // close collection and indexes - collectionOperations.close(); - // drop collection and indexes collectionOperations.dropCollection(); } @@ -337,12 +318,22 @@ public void drop() { isDropped = true; } + public boolean isDropped() { + try { + readLock.lock(); + return isDropped || nitriteMap == null || nitriteMap.isDropped(); + } finally { + readLock.unlock(); + } + } + public boolean isOpen() { try { readLock.lock(); - return nitriteStore != null && !nitriteStore.isClosed() && !isDropped; + return nitriteStore != null && !nitriteStore.isClosed() + && !isDropped && !nitriteMap.isClosed() && !nitriteMap.isDropped(); } catch (Exception e) { - throw new NitriteIOException("failed to close the database", e); + throw new NitriteIOException("Failed to check the collection state", e); } finally { readLock.unlock(); } @@ -455,15 +446,15 @@ private void initialize() { private void checkOpened() { if (isOpen()) return; - throw new NitriteIOException("collection is closed"); + throw new NitriteIOException("Collection is closed"); } private void validateRebuildIndex(IndexDescriptor indexDescriptor) { notNull(indexDescriptor, "index cannot be null"); - String[] indexFields = indexDescriptor.getIndexFields().getFieldNames().toArray(new String[0]); + String[] indexFields = indexDescriptor.getFields().getFieldNames().toArray(new String[0]); if (isIndexing(indexFields)) { - throw new IndexingException("indexing on value " + indexDescriptor.getIndexFields() + " is currently running"); + throw new IndexingException("Cannot rebuild index, index is currently being built"); } } diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/Document.java b/nitrite/src/main/java/org/dizitart/no2/collection/Document.java index 871600e46..9b47640c3 100644 --- a/nitrite/src/main/java/org/dizitart/no2/collection/Document.java +++ b/nitrite/src/main/java/org/dizitart/no2/collection/Document.java @@ -151,13 +151,21 @@ static Document createDocument(Map documentMap) { Document merge(Document update); /** - * Checks if a key exists in the document. + * Checks if a top level key exists in the document. * * @param key the key * @return the boolean */ boolean containsKey(String key); + /** + * Checks if a top level field or embedded field exists in the document. + * + * @param field the field + * @return the boolean + */ + boolean containsField(String field); + /** * Gets the document revision number. * diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/DocumentCursor.java b/nitrite/src/main/java/org/dizitart/no2/collection/DocumentCursor.java index ea0dacca3..ab87832ea 100644 --- a/nitrite/src/main/java/org/dizitart/no2/collection/DocumentCursor.java +++ b/nitrite/src/main/java/org/dizitart/no2/collection/DocumentCursor.java @@ -20,13 +20,10 @@ import org.dizitart.no2.common.RecordStream; /** - * An interface to iterate over database {@code find()} results. It provides a + * An interface to iterate over {@link NitriteCollection#find()} results. It provides a * mechanism to iterate over all {@link NitriteId}s of the result. * 

  * {@code
- * // create/open a database
- * Nitrite db = Nitrite.builder()
- *      .openOrCreate("user", "password");
  *
  * // create/open a database
  * Nitrite db = Nitrite.builder()
@@ -67,8 +64,8 @@ public interface DocumentCursor extends RecordStream {
     /**
      * Performs a left outer join with a foreign cursor with the specified lookup parameters.
      * 

-     * It performs an equality match on the localString to the foreignString from the documents of the foreign cursor.
-     * If an input document does not contain the localString, the join treats the field as having a value of `null`
+     * It performs an equality match on the localField to the foreignField from the documents of the foreign cursor.
+     * If an input document does not contain the localField, the join treats the field as having a value of null
      * for matching purposes.
      *
      * @param foreignCursor the foreign cursor for the join.
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/FindOptions.java b/nitrite/src/main/java/org/dizitart/no2/collection/FindOptions.java
index 56be8bdcd..45469e068 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/FindOptions.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/FindOptions.java
@@ -39,10 +39,12 @@ public class FindOptions {
     private SortableFields orderBy;
     private Long skip;
     private Long limit;
+    private boolean distinct = false;
 
     /**
      * Specifies the {@link Collator}.
      *
+     * @param collator the collator
      * @return the collator.
      */
     @Setter(AccessLevel.PUBLIC)
@@ -95,6 +97,15 @@ public static FindOptions limitBy(long limit) {
         return findOptions;
     }
 
+    /**
+     * Indicates if the find operation should return distinct results.
+     */
+    public static FindOptions withDistinct() {
+        FindOptions findOptions = new FindOptions();
+        findOptions.distinct(true);
+        return findOptions;
+    }
+
     /**
      * Skip find options.
      *
@@ -156,4 +167,15 @@ public FindOptions thenOrderBy(String fieldName, SortOrder sortOrder) {
         }
         return this;
     }
+
+    /**
+     * Indicates if the find operation should return distinct and unique results.
+     *
+     * @param distinct the distinct
+     * @return the find options
+     */
+    public FindOptions withDistinct(boolean distinct) {
+        this.distinct = distinct;
+        return this;
+    }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/FindPlan.java b/nitrite/src/main/java/org/dizitart/no2/collection/FindPlan.java
index d2daeec01..a2f86a27d 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/FindPlan.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/FindPlan.java
@@ -20,7 +20,7 @@
 import lombok.Data;
 import org.dizitart.no2.common.SortOrder;
 import org.dizitart.no2.common.tuples.Pair;
-import org.dizitart.no2.filters.EqualsFilter;
+import org.dizitart.no2.filters.FieldBasedFilter;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.filters.IndexScanFilter;
 import org.dizitart.no2.index.IndexDescriptor;
@@ -38,7 +38,7 @@
  */
 @Data
 public class FindPlan {
-    private EqualsFilter byIdFilter;
+    private FieldBasedFilter byIdFilter;
     private IndexScanFilter indexScanFilter;
     private Filter collectionScanFilter;
 
@@ -48,6 +48,7 @@ public class FindPlan {
 
     private Long skip;
     private Long limit;
+    private boolean distinct;
 
     private Collator collator;
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/NitriteCollection.java b/nitrite/src/main/java/org/dizitart/no2/collection/NitriteCollection.java
index 92bd6dfb8..01a721198 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/NitriteCollection.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/NitriteCollection.java
@@ -46,7 +46,6 @@
  * 
  * {@code
  * Nitrite db = Nitrite.builder()
- *    .loadModule(MVStoreModule("/tmp/tmp.db"))
  *    .openOrCreate("user", "password");
  *    
  * NitriteCollection collection = db.getCollection("products");
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/NitriteDocument.java b/nitrite/src/main/java/org/dizitart/no2/collection/NitriteDocument.java
index 95e418649..84e51e43d 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/NitriteDocument.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/NitriteDocument.java
@@ -59,17 +59,17 @@ class NitriteDocument extends LinkedHashMap implements Document
     public Document put(String field, Object value) {
         // field name cannot be empty or null
         if (isNullOrEmpty(field)) {
-            throw new InvalidOperationException("document does not support empty or null key");
+            throw new InvalidOperationException("Document does not support empty or null key");
         }
 
-        // _id field can not be set manually
+        // field name cannot be empty or null
         if (DOC_ID.contentEquals(field) && !validId(value)) {
             throw new InvalidOperationException("_id is an auto generated value and cannot be set");
         }
 
         // value must be serializable
         if (value != null && !Serializable.class.isAssignableFrom(value.getClass())) {
-            throw new ValidationException("type " + value.getClass().getName()
+            throw new ValidationException("Type " + value.getClass().getName()
                 + " does not implement java.io.Serializable");
         }
 
@@ -118,7 +118,7 @@ public NitriteId getId() {
             // create a nitrite id instance from the string value
             return createId(id);
         } catch (ClassCastException cce) {
-            throw new InvalidIdException("invalid _id found " + get(DOC_ID));
+            throw new InvalidIdException("Invalid _id found " + get(DOC_ID));
         }
     }
 
@@ -170,16 +170,28 @@ public Document clone() {
     public Document merge(Document document) {
         if (document instanceof NitriteDocument) {
             super.putAll((NitriteDocument) document);
+        } else {
+            throw new InvalidOperationException("Document merge only supports NitriteDocument");
         }
         return this;
     }
 
     @Override
     public boolean containsKey(String key) {
-//        return getFields().contains(key);
         return super.containsKey(key);
     }
 
+    @Override
+    public boolean containsField(String field) {
+        if (containsKey(field)) {
+            // search top level
+            return true;
+        } else {
+            // search deep level
+            return getFields().contains(field);
+        }
+    }
+
     @Override
     public boolean equals(Object other) {
         if (other == this)
@@ -212,6 +224,11 @@ public boolean equals(Object other) {
         return true;
     }
 
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+
     @Override
     public Iterator> iterator() {
         return new PairIterator(super.entrySet().iterator());
@@ -262,7 +279,7 @@ private Object deepGet(String field) {
 
     private void deepPut(String[] splits, Object value) {
         if (splits.length == 0) {
-            throw new ValidationException("invalid key provided");
+            throw new ValidationException("Invalid key provided");
         }
         String key = splits[0];
         if (splits.length == 1) {
@@ -292,7 +309,7 @@ private void deepPut(String[] splits, Object value) {
 
     private void deepRemove(String[] splits) {
         if (splits.length == 0) {
-            throw new ValidationException("invalid key provided");
+            throw new ValidationException("Invalid key provided");
         }
         String key = splits[0];
         if (splits.length == 1) {
@@ -314,8 +331,42 @@ private void deepRemove(String[] splits) {
                     // remove the current level document also
                     super.remove(key);
                 }
-            } else if (val == null) {
-                // if current level value is null, remove the key
+            } else if (val instanceof List && isInteger(splits[1])) {
+                // if the current level value is an iterable,
+                // remove the element at the next level
+                List list = (List) val;
+                int index = Integer.parseInt(splits[1]);
+                Object item = list.get(index);
+                if (splits.length > 2 && item instanceof NitriteDocument) {
+                    // if there are more splits, then this is an embedded document
+                    // so remove the element at the next level
+                    ((NitriteDocument) item).deepRemove(Arrays.copyOfRange(splits, 2, splits.length));
+                } else {
+                    // if there are no more splits, then this is a primitive value
+                    // so remove the element at the next level
+                    list.remove(index);
+                    this.put(key, list);
+                }
+            } else if (val != null && val.getClass().isArray()) {
+                // if the current level value is an array,
+                // remove the element at the next level
+                Object[] array = convertToObjectArray(val);
+                int index = Integer.parseInt(splits[1]);
+                Object item = array[index];
+                if (splits.length > 2 && item instanceof NitriteDocument) {
+                    // if there are more splits, then this is an embedded document
+                    // so remove the element at the next level
+                    ((NitriteDocument) item).deepRemove(Arrays.copyOfRange(splits, 2, splits.length));
+                } else {
+                    // if there are no more splits, then this is a primitive value
+                    // so remove the element at the next level
+                    List list = Arrays.asList(array);
+                    list.remove(index);
+                    this.put(key, list.toArray());
+                }
+            } else {
+                // if current level value is not an iterable,
+                // remove the key
                 super.remove(key);
             }
         }
@@ -365,15 +416,9 @@ private Object recursiveGet(Object object, String[] remainingPath) {
                 // convert the key as an integer index
                 int index = asInteger(accessor);
 
-                // check index lower bound
-                if (index < 0) {
-                    throw new ValidationException("invalid array index " + index + " to access item inside a document");
-                }
-
-                // check index upper bound
-                if (index >= array.length) {
-                    throw new ValidationException("index " + index +
-                        " is not less than the size of the array " + array.length);
+                // check index bound
+                if (index < 0 || index >= array.length) {
+                    throw new ValidationException("Invalid index " + index + " to access item inside a document");
                 }
 
                 // get the value at the index from the array
@@ -406,15 +451,9 @@ private Object recursiveGet(Object object, String[] remainingPath) {
                 // convert the key as an integer index
                 int index = asInteger(accessor);
 
-                // check index lower bound
-                if (index < 0) {
-                    throw new ValidationException("invalid collection index " + index + " to access item inside a document");
-                }
-
-                // check index upper bound
-                if (index >= collection.size()) {
-                    throw new ValidationException("index " + accessor +
-                        " is not less than the size of the list " + collection.size());
+                // check index bound
+                if (index < 0 || index >= collection.size()) {
+                    throw new ValidationException("Invalid index " + index + " to access item inside a document");
                 }
 
                 // get the value at the index from the list
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/NitriteId.java b/nitrite/src/main/java/org/dizitart/no2/collection/NitriteId.java
index 850505a56..302053861 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/NitriteId.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/NitriteId.java
@@ -28,10 +28,10 @@
 import static org.dizitart.no2.common.Constants.ID_SUFFIX;
 
 /**
- * An unique identifier across the Nitrite database. Each document in
+ * A unique identifier across the Nitrite database. Each document in
  * a nitrite collection is associated with a {@link NitriteId}.
  * 

- * During insertion if an unique object is supplied in the '_id' field
+ * During insertion if a unique object is supplied in the '_id' field
  * of the document, then the value of the '_id' field will be used to
  * create a new {@link NitriteId}. If that is not supplied, then nitrite
  * will auto generate one and supply it in the '_id' field of the document.
@@ -43,7 +43,7 @@
 @EqualsAndHashCode
 public final class NitriteId implements Comparable, Serializable {
     private static final long serialVersionUID = 1477462375L;
-    private transient static final SnowflakeIdGenerator generator = new SnowflakeIdGenerator();
+    private static final SnowflakeIdGenerator generator = new SnowflakeIdGenerator();
 
     private String idValue;
 
@@ -83,14 +83,14 @@ public static boolean validId(Object value) {
             Long.parseLong(value.toString());
             return true;
         } catch (Exception e) {
-            throw new InvalidIdException("id must be a string representation of 64bit decimal number");
+            throw new InvalidIdException("id must be a string representation of 64bit integer number " + value);
         }
     }
 
     @Override
     public int compareTo(NitriteId other) {
         if (other.idValue == null) {
-            throw new InvalidIdException("cannot compare with null id");
+            throw new InvalidIdException("Cannot compare with null id");
         }
 
         return Long.compare(Long.parseLong(idValue), Long.parseLong(other.idValue));
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/SnowflakeIdGenerator.java b/nitrite/src/main/java/org/dizitart/no2/collection/SnowflakeIdGenerator.java
index 9ef5c1b73..a7b56663c 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/SnowflakeIdGenerator.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/SnowflakeIdGenerator.java
@@ -19,11 +19,9 @@
 
 import lombok.extern.slf4j.Slf4j;
 
-import java.net.NetworkInterface;
-import java.net.SocketException;
+import java.nio.ByteBuffer;
 import java.security.SecureRandom;
-import java.util.Enumeration;
-import java.util.NoSuchElementException;
+import java.util.UUID;
 
 /**
  * Generate unique IDs using the Twitter Snowflake algorithm (see https://github.com/twitter/snowflake). Snowflake IDs
@@ -51,23 +49,19 @@ public class SnowflakeIdGenerator {
 
     private volatile long lastTimestamp = -1L;
     private volatile long sequence = 0L;
+    private static final long no2epoch = 1288834974657L;
 
 
     public SnowflakeIdGenerator() {
         random = new SecureRandom();
         long maxNodeId = ~(-1L << nodeIdBits);
-        try {
-            this.nodeId = getNodeId();
-        } catch (SocketException | NoSuchElementException | NullPointerException e) {
-            log.warn("SNOWFLAKE: could not determine machine address; using random node id");
-            this.nodeId = random.nextInt((int) maxNodeId) + 1;
-        }
+        this.nodeId = getNodeId();
 
         if (this.nodeId > maxNodeId) {
-            log.warn("SNOWFLAKE: nodeId > maxNodeId; using random node id");
+            log.warn("nodeId > maxNodeId; using random node id");
             this.nodeId = random.nextInt((int) maxNodeId) + 1;
         }
-        log.debug("SNOWFLAKE: initialised with node id {}", this.nodeId);
+        log.debug("initialised with node id {}", this.nodeId);
     }
 
     protected long tillNextMillis(long lastTimestamp) {
@@ -78,27 +72,11 @@ protected long tillNextMillis(long lastTimestamp) {
         return timestamp;
     }
 
-    protected long getNodeId() throws SocketException {
-        NetworkInterface network = null;
-
-        Enumeration en = NetworkInterface.getNetworkInterfaces();
-        while (en.hasMoreElements()) {
-            NetworkInterface nint = en.nextElement();
-            if (!nint.isLoopback() && nint.getHardwareAddress() != null) {
-                network = nint;
-                break;
-            }
-        }
-
-        if (network != null) {
-            byte[] mac = network.getHardwareAddress();
-            byte rndByte = (byte) (random.nextInt() & 0x000000FF);
+    protected long getNodeId() {
+        byte[] uuid = asBytes(UUID.randomUUID());
+        byte rndByte = (byte) (random.nextInt() & 0x000000FF);
 
-            // take the last byte of the MAC address and a random byte as node id
-            return ((0x000000FF & (long) mac[mac.length - 1]) | (0x0000FF00 & (((long) rndByte) << 8))) >> 6;
-        } else {
-            throw new NoSuchElementException("no network interface found");
-        }
+        return ((0x000000FF & (long) uuid[uuid.length - 1]) | (0x0000FF00 & (((long) rndByte) << 8))) >> 6;
     }
 
 
@@ -128,12 +106,18 @@ public synchronized long getId() {
         }
         lastTimestamp = timestamp;
         long timestampLeftShift = sequenceBits + nodeIdBits;
-        long twepoch = 1288834974657L;
-        long id = ((timestamp - twepoch) << timestampLeftShift) | (nodeId << sequenceBits) | sequence;
+        long id = ((timestamp - no2epoch) << timestampLeftShift) | (nodeId << sequenceBits) | sequence;
 
         if (id < 0) {
-            log.warn("Id is smaller than 0: {}", id);
+            log.warn("Generated id is negative: {}", id);
         }
         return id;
     }
+
+    private byte[] asBytes(UUID uuid) {
+        ByteBuffer bb = ByteBuffer.wrap(new byte[16]);
+        bb.putLong(uuid.getMostSignificantBits());
+        bb.putLong(uuid.getLeastSignificantBits());
+        return bb.array();
+    }
 }
\ No newline at end of file
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/CollectionOperations.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/CollectionOperations.java
index 836bdca44..fb8354ebd 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/CollectionOperations.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/CollectionOperations.java
@@ -20,22 +20,19 @@
 import org.dizitart.no2.collection.*;
 import org.dizitart.no2.collection.events.CollectionEventInfo;
 import org.dizitart.no2.collection.events.CollectionEventListener;
-import org.dizitart.no2.collection.meta.Attributes;
+import org.dizitart.no2.common.meta.Attributes;
 import org.dizitart.no2.common.Fields;
 import org.dizitart.no2.common.WriteResult;
 import org.dizitart.no2.common.event.EventBus;
-import org.dizitart.no2.filters.Filter;
-import org.dizitart.no2.index.IndexDescriptor;
 import org.dizitart.no2.common.processors.Processor;
 import org.dizitart.no2.common.processors.ProcessorChain;
+import org.dizitart.no2.filters.Filter;
+import org.dizitart.no2.index.IndexDescriptor;
 import org.dizitart.no2.store.NitriteMap;
 import org.dizitart.no2.store.StoreCatalog;
 
 import java.util.Collection;
 
-import static org.dizitart.no2.collection.UpdateOptions.updateOptions;
-import static org.dizitart.no2.common.util.DocumentUtils.createUniqueFilter;
-
 /**
  * The collection operations.
  *
@@ -77,20 +74,9 @@ public CollectionOperations(String collectionName,
      * @param processor the processor
      */
     public void addProcessor(Processor processor) {
-        doProcess(processor);
         processorChain.add(processor);
     }
 
-    /**
-     * Removes a document processor.
-     *
-     * @param processor the processor
-     */
-    public void removeProcessor(Processor processor) {
-        processorChain.remove(processor);
-        undoProcess(processor);
-    }
-
     /**
      * Creates index.
      *
@@ -271,6 +257,11 @@ public void close() {
         nitriteMap.close();
     }
 
+    public void clear() {
+        nitriteMap.clear();
+        indexOperations.clear();
+    }
+
     private void initialize() {
         this.processorChain = new ProcessorChain();
         this.indexOperations = new IndexOperations(collectionName, nitriteConfig, nitriteMap, eventBus);
@@ -290,18 +281,4 @@ private void dropNitriteMap() {
         // drop the map
         nitriteMap.drop();
     }
-
-    private void doProcess(Processor processor) {
-        for (Document document : find(Filter.ALL, null)) {
-            Document processed = processor.processBeforeWrite(document);
-            update(createUniqueFilter(document), processed, updateOptions(false));
-        }
-    }
-
-    private void undoProcess(Processor processor) {
-        for (Document document : find(Filter.ALL, null)) {
-            Document processed = processor.processAfterRead(document);
-            update(createUniqueFilter(document), processed, updateOptions(false));
-        }
-    }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/DocumentIndexWriter.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/DocumentIndexWriter.java
index 394cfb706..85ec0ccc8 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/DocumentIndexWriter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/DocumentIndexWriter.java
@@ -81,7 +81,7 @@ void updateIndexEntry(Document oldDocument, Document newDocument) {
     private void writeIndexEntryInternal(IndexDescriptor indexDescriptor, Document document,
                                          NitriteIndexer nitriteIndexer) {
         if (indexDescriptor != null) {
-            Fields fields = indexDescriptor.getIndexFields();
+            Fields fields = indexDescriptor.getFields();
             FieldValues fieldValues = DocumentUtils.getValues(document, fields);
 
             // if dirty index and currently indexing is not running, rebuild
@@ -97,7 +97,7 @@ private void writeIndexEntryInternal(IndexDescriptor indexDescriptor, Document d
     private void removeIndexEntryInternal(IndexDescriptor indexDescriptor, Document document,
                                           NitriteIndexer nitriteIndexer) {
         if (indexDescriptor != null) {
-            Fields fields = indexDescriptor.getIndexFields();
+            Fields fields = indexDescriptor.getFields();
             FieldValues fieldValues = DocumentUtils.getValues(document, fields);
 
             // if dirty index and currently indexing is not running, rebuild
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/FindOptimizer.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/FindOptimizer.java
index 58b129016..43a44b11a 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/FindOptimizer.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/FindOptimizer.java
@@ -49,6 +49,7 @@ public FindPlan optimize(Filter filter,
 
         if (findOptions != null) {
             findPlan.setCollator(findOptions.collator());
+            findPlan.setDistinct(findOptions.distinct());
         }
         return findPlan;
     }
@@ -178,7 +179,7 @@ private void planForIndexOnlyFilters(FindPlan findPlan, Set in
                     // if filter is compatible with already identified index only filter then add
                     indexOnlyFilters.add(indexScanFilter);
                 } else {
-                    throw new FilterException("a query can not have multiple index only filters");
+                    throw new FilterException("A query can not have multiple index only filters");
                 }
             }
         }
@@ -221,7 +222,7 @@ private void planForIndexScanningFilters(FindPlan findPlan, Set> indexFilterMap = new TreeMap<>(Collections.reverseOrder());
 
         for (IndexDescriptor indexDescriptor : indexDescriptors) {
-            List fieldNames = indexDescriptor.getIndexFields().getFieldNames();
+            List fieldNames = indexDescriptor.getFields().getFieldNames();
 
             List indexedFilters = new ArrayList<>();
             for (String fieldName : fieldNames) {
@@ -283,7 +284,7 @@ private void planForCollectionScanningFilters(FindPlan findPlan, Set filters) {
         for (Filter filter : filters) {
             if (filter instanceof IndexOnlyFilter) {
-                throw new FilterException("collection scan is not supported for the filter " + filter);
+                throw new FilterException("Collection scan is not supported for the filter " + filter);
             } else if (filter instanceof TextFilter) {
                 throw new FilterException(((TextFilter) filter).getField() + " is not full-text indexed");
             }
@@ -298,7 +299,7 @@ private void readSortOption(FindOptions findOptions, FindPlan findPlan) {
 
             if (indexDescriptor != null) {
                 // get index field names
-                List indexedFieldNames = indexDescriptor.getIndexFields().getFieldNames();
+                List indexedFieldNames = indexDescriptor.getFields().getFieldNames();
 
                 boolean canUseIndex = false;
                 Map indexScanOrder = new HashMap<>();
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexManager.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexManager.java
index 2464048e2..96bee500b 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexManager.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexManager.java
@@ -84,7 +84,7 @@ public Collection findMatchingIndexDescriptors(Fields fields) {
         List indexDescriptors = new ArrayList<>();
 
         for (IndexDescriptor indexDescriptor : getIndexDescriptors()) {
-            if (indexDescriptor.getIndexFields().startsWith(fields)) {
+            if (indexDescriptor.getFields().startsWith(fields)) {
                 indexDescriptors.add(indexDescriptor);
             }
         }
@@ -103,17 +103,33 @@ public IndexDescriptor findExactIndexDescriptor(Fields fields) {
     @Override
     public void close() {
         // close all index maps
-        Iterable indexMetas = indexMetaMap.values();
-        for (IndexMeta indexMeta : indexMetas) {
-            if (indexMeta != null && indexMeta.getIndexDescriptor() != null) {
-                String indexMapName = indexMeta.getIndexMap();
-                NitriteMap indexMap = nitriteStore.openMap(indexMapName, Object.class, Object.class);
-                indexMap.close();
+        if (!indexMetaMap.isClosed() && !indexMetaMap.isDropped()) {
+            Iterable indexMetas = indexMetaMap.values();
+            for (IndexMeta indexMeta : indexMetas) {
+                if (indexMeta != null && indexMeta.getIndexDescriptor() != null) {
+                    String indexMapName = indexMeta.getIndexMap();
+                    NitriteMap indexMap = nitriteStore.openMap(indexMapName, Object.class, Object.class);
+                    indexMap.close();
+                }
             }
+
+            // close index meta
+            indexMetaMap.close();
         }
+    }
 
-        // close index meta
-        indexMetaMap.close();
+    public void clearAll() {
+        // close all index maps
+        if (!indexMetaMap.isClosed() && !indexMetaMap.isDropped()) {
+            Iterable indexMetas = indexMetaMap.values();
+            for (IndexMeta indexMeta : indexMetas) {
+                if (indexMeta != null && indexMeta.getIndexDescriptor() != null) {
+                    String indexMapName = indexMeta.getIndexMap();
+                    NitriteMap indexMap = nitriteStore.openMap(indexMapName, Object.class, Object.class);
+                    indexMap.clear();
+                }
+            }
+        }
     }
 
     /**
@@ -181,7 +197,6 @@ void dropIndexDescriptor(Fields fields) {
     }
 
     void dropIndexMeta() {
-        indexMetaMap.clear();
         indexMetaMap.drop();
     }
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexOperations.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexOperations.java
index 9d6dc9a45..56441ad32 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexOperations.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/IndexOperations.java
@@ -61,7 +61,7 @@ void createIndex(Fields fields, String indexType) {
             indexDescriptor = indexManager.createIndexDescriptor(fields, indexType);
         } else {
             // if index already there throw
-            throw new IndexingException("index already exists on " + fields);
+            throw new IndexingException("Index already exists on fields: " + fields);
         }
 
         buildIndex(indexDescriptor, false);
@@ -70,17 +70,17 @@ void createIndex(Fields fields, String indexType) {
     // call to this method is already synchronized, only one thread per field
     // can access it only if rebuild is already not running for that field
     void buildIndex(IndexDescriptor indexDescriptor, boolean rebuild) {
-        final Fields fields = indexDescriptor.getIndexFields();
+        final Fields fields = indexDescriptor.getFields();
         if (getBuildFlag(fields).compareAndSet(false, true)) {
             buildIndexInternal(indexDescriptor, rebuild);
             return;
         }
-        throw new IndexingException("indexing is already running on " + indexDescriptor.getIndexFields());
+        throw new IndexingException("Index build already in progress on fields: " + indexDescriptor.getFields());
     }
 
     void dropIndex(Fields fields) {
         if (getBuildFlag(fields).get()) {
-            throw new IndexingException("cannot drop index as indexing is running on " + fields);
+            throw new IndexingException("Index build already in progress on fields: " + fields);
         }
 
         IndexDescriptor indexDescriptor = findIndexDescriptor(fields);
@@ -92,39 +92,51 @@ void dropIndex(Fields fields) {
             indexManager.dropIndexDescriptor(fields);
             indexBuildTracker.remove(fields);
         } else {
-            throw new IndexingException(fields + " is not indexed");
+            throw new IndexingException("Index does not exist on fields: " + fields);
         }
     }
 
     void dropAllIndices() {
         for (Map.Entry entry : indexBuildTracker.entrySet()) {
             if (entry.getValue() != null && entry.getValue().get()) {
-                throw new IndexingException("cannot drop index as indexing is running on " + entry.getKey());
+                throw new IndexingException("Index build already in progress on fields: " + entry.getKey());
             }
         }
 
         // we can drop all indices in parallel
         List> futures = new ArrayList<>();
         for (IndexDescriptor index : listIndexes()) {
-            futures.add(runAsync(() -> dropIndex(index.getIndexFields())));
+            futures.add(runAsync(() -> dropIndex(index.getFields())));
         }
 
         for (Future future : futures) {
             try {
                 future.get();
             } catch (InterruptedException | ExecutionException e) {
-                throw new IndexingException("failed to drop all indices", e);
+                throw new IndexingException("Failed to drop all indices", e);
             }
         }
 
         indexManager.dropIndexMeta();
         indexBuildTracker.clear();
+        indexManager.close();
 
         // recreate index manager to discard old native resources
         // special measure for RocksDB adapter
         this.indexManager = new IndexManager(collectionName, nitriteConfig);
     }
 
+    void clear() {
+        for (Map.Entry entry : indexBuildTracker.entrySet()) {
+            if (entry.getValue() != null && entry.getValue().get()) {
+                throw new IndexingException("Index build already in progress on fields: " + entry.getKey());
+            }
+        }
+
+        indexManager.clearAll();
+        indexBuildTracker.clear();
+    }
+
     boolean isIndexing(Fields field) {
         // has an index will only return true, if there is an index on
         // the value and indexing is not running on it
@@ -144,7 +156,11 @@ IndexDescriptor findIndexDescriptor(Fields field) {
         return indexManager.findExactIndexDescriptor(field);
     }
 
-    AtomicBoolean getBuildFlag(Fields field) {
+    boolean shouldRebuildIndex(Fields fields) {
+        return indexManager.isDirtyIndex(fields) && !getBuildFlag(fields).get();
+    }
+
+    private AtomicBoolean getBuildFlag(Fields field) {
         AtomicBoolean flag = indexBuildTracker.get(field);
         if (flag != null) return flag;
 
@@ -153,12 +169,8 @@ AtomicBoolean getBuildFlag(Fields field) {
         return flag;
     }
 
-    boolean shouldRebuildIndex(Fields fields) {
-        return indexManager.isDirtyIndex(fields) && !getBuildFlag(fields).get();
-    }
-
     private void buildIndexInternal(IndexDescriptor indexDescriptor, boolean rebuild) {
-        Fields fields = indexDescriptor.getIndexFields();
+        Fields fields = indexDescriptor.getFields();
         try {
             alert(EventType.IndexStart, fields);
             // first put dirty marker
@@ -174,12 +186,12 @@ private void buildIndexInternal(IndexDescriptor indexDescriptor, boolean rebuild
 
             for (Pair entry : nitriteMap.entries()) {
                 Document document = entry.getSecond();
-                FieldValues fieldValues = DocumentUtils.getValues(document, indexDescriptor.getIndexFields());
+                FieldValues fieldValues = DocumentUtils.getValues(document, indexDescriptor.getFields());
                 nitriteIndexer.writeIndexEntry(fieldValues, indexDescriptor, nitriteConfig);
             }
         } finally {
             // remove dirty marker to denote indexing completed successfully
-            // if dirty marker is found in any index, it needs to be rebuild
+            // if dirty marker is found in any index, it needs to be rebuilt
             indexManager.endIndexing(fields);
             getBuildFlag(fields).set(false);
             alert(EventType.IndexEnd, fields);
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java
index 4f7411feb..e9ac800d3 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java
@@ -21,10 +21,7 @@
 import org.dizitart.no2.common.RecordStream;
 import org.dizitart.no2.common.streams.*;
 import org.dizitart.no2.common.tuples.Pair;
-import org.dizitart.no2.filters.EqualsFilter;
-import org.dizitart.no2.filters.Filter;
-import org.dizitart.no2.filters.LogicalFilter;
-import org.dizitart.no2.filters.NitriteFilter;
+import org.dizitart.no2.filters.*;
 import org.dizitart.no2.index.IndexDescriptor;
 import org.dizitart.no2.index.NitriteIndexer;
 import org.dizitart.no2.common.processors.ProcessorChain;
@@ -73,7 +70,11 @@ public DocumentCursor find(Filter filter, FindOptions findOptions) {
     }
 
     Document getById(NitriteId nitriteId) {
-        return nitriteMap.get(nitriteId);
+        Document document = nitriteMap.get(nitriteId);
+        if (processorChain != null) {
+            document = processorChain.processAfterRead(document);
+        }
+        return document;
     }
 
     private void prepareFilter(Filter filter) {
@@ -104,6 +105,13 @@ private void prepareLogicalFilter(LogicalFilter logicalFilter) {
         }
     }
 
+    private DocumentCursor createCursor(FindPlan findPlan) {
+        RecordStream> recordStream = findSuitableStream(findPlan);
+        DocumentStream cursor = new DocumentStream(recordStream, processorChain);
+        cursor.setFindPlan(findPlan);
+        return cursor;
+    }
+
     private RecordStream> findSuitableStream(FindPlan findPlan) {
         RecordStream> rawStream;
 
@@ -114,17 +122,21 @@ private RecordStream> findSuitableStream(FindPlan find
                 RecordStream> suitableStream = findSuitableStream(subPlan);
                 subStreams.add(suitableStream);
             }
-            // union of all suitable stream of all sub plans
-            rawStream = new UnionStream(subStreams);
 
-            // return only distinct items
-            rawStream = new DistinctStream(rawStream);
+            // concat all suitable stream of all sub plans
+            rawStream = new ConcatStream(subStreams);
+
+            if (findPlan.isDistinct()) {
+                // return only distinct items
+                rawStream = new DistinctStream(rawStream);
+            }
         } else {
             // and or single filter
             if (findPlan.getByIdFilter() != null) {
-                EqualsFilter byIdFilter = findPlan.getByIdFilter();
+                FieldBasedFilter byIdFilter = findPlan.getByIdFilter();
                 NitriteId nitriteId = NitriteId.createId((String) byIdFilter.getValue());
-                rawStream = RecordStream.single(pair(nitriteId, nitriteMap.get(nitriteId)));
+                Document document = nitriteMap.get(nitriteId);
+                rawStream = RecordStream.single(pair(nitriteId, document));
             } else {
                 IndexDescriptor indexDescriptor = findPlan.getIndexDescriptor();
                 if (indexDescriptor != null) {
@@ -153,17 +165,10 @@ private RecordStream> findSuitableStream(FindPlan find
             if (findPlan.getLimit() != null || findPlan.getSkip() != null) {
                 long limit = findPlan.getLimit() == null ? Long.MAX_VALUE : findPlan.getLimit();
                 long skip = findPlan.getSkip() == null ? 0 : findPlan.getSkip();
-                rawStream = new BoundedDocumentStream(skip, limit, rawStream);
+                rawStream = new BoundedStream<>(skip, limit, rawStream);
             }
         }
 
         return rawStream;
     }
-
-    private DocumentCursor createCursor(FindPlan findPlan) {
-        RecordStream> recordStream = findSuitableStream(findPlan);
-        DocumentStream cursor = new DocumentStream(recordStream, processorChain);
-        cursor.setFindPlan(findPlan);
-        return cursor;
-    }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/WriteOperations.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/WriteOperations.java
index ca0727f86..9f491055b 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/WriteOperations.java
+++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/WriteOperations.java
@@ -85,22 +85,20 @@ WriteResult insert(Document... documents) {
             // run processors
             Document unprocessed = newDoc.clone();
             Document processed = processorChain.processBeforeWrite(unprocessed);
-            log.debug("Document processed from {} to {} before insert", newDoc, processed);
+            log.debug("Processed document with id: {}", nitriteId);
 
-            log.debug("Inserting processed document {} in {}", processed, nitriteMap.getName());
+            log.debug("Inserting processed document with id {}", nitriteId);
             Document already = nitriteMap.putIfAbsent(nitriteId, processed);
 
             if (already != null) {
-                log.warn("Another document {} already exists with same id {}", already, nitriteId);
-
-                throw new UniqueConstraintException("id constraint violation, " +
-                    "entry with same id already exists in " + nitriteMap.getName());
+                throw new UniqueConstraintException("Document with id " + nitriteId + " already exists" +
+                    " in " + nitriteMap.getName());
             } else {
                 try {
                     documentIndexWriter.writeIndexEntry(processed);
                 } catch (UniqueConstraintException | IndexingException e) {
-                    log.error("Index operation has failed during insertion for the document "
-                        + document + " in " + nitriteMap.getName(), e);
+                    log.error("Error while writing index entry for document with id : {} in {}",
+                        nitriteId, nitriteMap.getName(), e);
                     nitriteMap.remove(nitriteId);
                     throw e;
                 }
@@ -113,7 +111,7 @@ WriteResult insert(Document... documents) {
             eventInfo.setTimestamp(time);
             eventInfo.setEventType(EventType.Insert);
             eventInfo.setOriginator(source);
-            alert(EventType.Insert, eventInfo);
+            alert(eventInfo);
         }
 
         WriteResultImpl result = new WriteResultImpl();
@@ -135,7 +133,7 @@ WriteResult update(Filter filter, Document update, UpdateOptions updateOptions)
         }
 
         if (document.size() == 0) {
-            alert(EventType.Update, new CollectionEventInfo<>());
+            log.debug("No fields to update");
             return writeResult;
         }
 
@@ -154,7 +152,7 @@ WriteResult update(Filter filter, Document update, UpdateOptions updateOptions)
                 long time = System.currentTimeMillis();
 
                 NitriteId nitriteId = newDoc.getId();
-                log.debug("Document to update {} in {}", newDoc, nitriteMap.getName());
+                log.debug("Updating document with id {} in {}", nitriteId, nitriteMap.getName());
 
                 if (!REPLICATOR.contentEquals(document.getSource())) {
                     document.remove(DOC_SOURCE);
@@ -170,21 +168,21 @@ WriteResult update(Filter filter, Document update, UpdateOptions updateOptions)
                 // run processor
                 Document unprocessed = newDoc.clone();
                 Document processed = processorChain.processBeforeWrite(unprocessed);
-                log.debug("Document processed from {} to {} before update", newDoc, processed);
+                log.debug("Processed document with id {}", nitriteId);
 
                 nitriteMap.put(nitriteId, processed);
-                log.debug("Document {} updated in {}", processed, nitriteMap.getName());
-
-                // if 'update' only contains id value, affected count = 0
-                if (document.size() > 0) {
-                    writeResult.addToList(nitriteId);
-                }
+                log.debug("Updated document with id {} in {}", nitriteId, nitriteMap.getName());
 
                 try {
                     documentIndexWriter.updateIndexEntry(oldDocument, processed);
+
+                    // if 'update' only contains id value, affected count = 0
+                    if (document.size() > 0) {
+                        writeResult.addToList(nitriteId);
+                    }
                 } catch (UniqueConstraintException | IndexingException e) {
-                    log.error("Index operation failed during update, reverting changes for the document "
-                        + oldDocument + " in " + nitriteMap.getName(), e);
+                    log.error("Error while writing index entry for document with id : {} in {}",
+                        nitriteId, nitriteMap.getName(), e);
                     nitriteMap.put(nitriteId, oldDocument);
                     documentIndexWriter.updateIndexEntry(processed, oldDocument);
                     throw e;
@@ -195,12 +193,12 @@ WriteResult update(Filter filter, Document update, UpdateOptions updateOptions)
                 eventInfo.setEventType(EventType.Update);
                 eventInfo.setTimestamp(time);
                 eventInfo.setOriginator(source);
-                alert(EventType.Update, eventInfo);
+                alert(eventInfo);
             }
         }
 
         if (count == 0) {
-            log.debug("No document found to update by the filter {} in {}", filter, nitriteMap.getName());
+            log.debug("No documents found for update in {}", nitriteMap.getName());
             if (updateOptions.isInsertIfAbsent()) {
                 return insert(update);
             } else {
@@ -208,8 +206,7 @@ WriteResult update(Filter filter, Document update, UpdateOptions updateOptions)
             }
         }
 
-        log.debug("Filter {} updated total {} document(s) with options {} in {}",
-            filter, count, updateOptions, nitriteMap.getName());
+        log.debug("Updated {} documents in {}", count, nitriteMap.getName());
 
         log.debug("Returning write result {} for collection {}", writeResult, nitriteMap.getName());
         return writeResult;
@@ -227,11 +224,11 @@ WriteResult remove(Filter filter, boolean justOnce) {
                 // run processor
                 Document unprocessed = document.clone();
                 Document processed = processorChain.processAfterRead(unprocessed);
-                log.debug("Document processed from {} to {} after remove", document, processed);
+                log.debug("Processed document with id : {}", processed.getId());
 
                 CollectionEventInfo eventInfo = removeAndCreateEvent(processed, result);
                 if (eventInfo != null) {
-                    alert(EventType.Remove, eventInfo);
+                    alert(eventInfo);
                 }
 
                 if (justOnce) {
@@ -241,14 +238,11 @@ WriteResult remove(Filter filter, boolean justOnce) {
         }
 
         if (count == 0) {
-            log.debug("No document found to remove by the filter {} in {}", filter, nitriteMap.getName());
+            log.debug("No documents found for filter {}", filter);
             return result;
         }
 
-        log.debug("Filter {} removed total {} document(s) with options {} from {}",
-            filter, count, justOnce, nitriteMap.getName());
-
-        log.debug("Returning write result {} for collection {}", result, nitriteMap.getName());
+        log.debug("Removed {} documents for filter : {}", count, filter);
         return result;
     }
 
@@ -257,7 +251,7 @@ WriteResult remove(Document document) {
         CollectionEventInfo eventInfo = removeAndCreateEvent(document, result);
         if (eventInfo != null) {
             eventInfo.setOriginator(document.getSource());
-            alert(EventType.Remove, eventInfo);
+            alert(eventInfo);
         }
         return result;
     }
@@ -266,28 +260,28 @@ private CollectionEventInfo removeAndCreateEvent(Document document, Wr
         NitriteId nitriteId = document.getId();
         document = nitriteMap.remove(nitriteId);
         if (document != null) {
-            long time = System.currentTimeMillis();
+            long removedAt = System.currentTimeMillis();
             documentIndexWriter.removeIndexEntry(document);
             writeResult.addToList(nitriteId);
 
             int rev = document.getRevision();
             document.put(DOC_REVISION, rev + 1);
-            document.put(DOC_MODIFIED, time);
+            document.put(DOC_MODIFIED, removedAt);
 
-            log.debug("Document removed {} from {}", document, nitriteMap.getName());
+            log.debug("Removed document with id {} from {}", document, nitriteMap.getName());
 
             CollectionEventInfo eventInfo = new CollectionEventInfo<>();
             Document eventDoc = document.clone();
             eventInfo.setItem(eventDoc);
             eventInfo.setEventType(EventType.Remove);
-            eventInfo.setTimestamp(time);
+            eventInfo.setTimestamp(removedAt);
             return eventInfo;
         }
         return null;
     }
 
-    private void alert(EventType action, CollectionEventInfo changedItem) {
-        log.debug("Notifying {} event for item {} from {}", action, changedItem, nitriteMap.getName());
+    private void alert(CollectionEventInfo changedItem) {
+        log.debug("Alerting event listeners for action : {} in {}", changedItem.getEventType(), nitriteMap.getName());
         if (eventBus != null) {
             eventBus.post(changedItem);
         }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/Constants.java b/nitrite/src/main/java/org/dizitart/no2/common/Constants.java
index dbae6f970..83118fe47 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/Constants.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/Constants.java
@@ -45,7 +45,7 @@ private Constants() {}
                 }
             }
         } catch (IOException e) {
-            throw new NitriteIOException("failed to load version information", e);
+            throw new NitriteIOException("Failed to load version information", e);
         }
         NITRITE_VERSION = v;
     }
@@ -138,7 +138,7 @@ private Constants() {}
     /**
      * The constant TAG_COLLECTION_METADATA.
      */
-    public static final String TAG_MAP_METADATA =  "mapNames";
+    public static final String TAG_MAP_METADATA = "mapNames";
 
     /**
      * The constant TAG_TYPE.
@@ -234,5 +234,4 @@ private Constants() {}
      * The constant INITIAL_REVISION.
      */
     public static final Integer INITIAL_SCHEMA_VERSION = 1;
-
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/DBNull.java b/nitrite/src/main/java/org/dizitart/no2/common/DBNull.java
index 48e8b7b37..93d6c143a 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/DBNull.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/DBNull.java
@@ -1,7 +1,5 @@
 package org.dizitart.no2.common;
 
-import org.dizitart.no2.index.DBValue;
-
 import java.io.Serializable;
 
 /**
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/DBValue.java b/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java
similarity index 95%
rename from nitrite/src/main/java/org/dizitart/no2/index/DBValue.java
rename to nitrite/src/main/java/org/dizitart/no2/common/DBValue.java
index 161b488ce..a1f67cd87 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/DBValue.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021 Nitrite author or authors.
+ * Copyright (c) 2017-2022 Nitrite author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
  *
  */
 
-package org.dizitart.no2.index;
+package org.dizitart.no2.common;
 
 import lombok.AccessLevel;
 import lombok.Data;
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/Fields.java b/nitrite/src/main/java/org/dizitart/no2/common/Fields.java
index e5f380868..bc89affb2 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/Fields.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/Fields.java
@@ -64,6 +64,9 @@ public static Fields withNames(String... fields) {
      * @return the fields
      */
     public Fields addField(String field) {
+        notNull(field, "field cannot be null");
+        notEmpty(field, "field cannot be empty");
+
         fieldNames.add(field);
         return this;
     }
@@ -78,28 +81,27 @@ public List getFieldNames() {
     }
 
     /**
-     * Starts with boolean.
+     * Check if a {@link Fields} is a subset of the current {@link Fields}.
      *
      * @param other the other
      * @return the boolean
      */
     public boolean startsWith(Fields other) {
-        if (other != null) {
-            int length = Math.min(fieldNames.size(), other.fieldNames.size());
+        notNull(other, "other cannot be null");
 
-            // if other is greater then it is not a prefix of this field
-            if (other.fieldNames.size() > length) return false;
+        int length = Math.min(fieldNames.size(), other.fieldNames.size());
 
-            for (int i = 0; i < length; i++) {
-                String thisField = fieldNames.get(i);
-                String otherField = other.fieldNames.get(i);
-                if (!thisField.equals(otherField)) {
-                    return false;
-                }
+        // if other is greater than it is not a prefix of this field
+        if (other.fieldNames.size() > length) return false;
+
+        for (int i = 0; i < length; i++) {
+            String thisField = fieldNames.get(i);
+            String otherField = other.fieldNames.get(i);
+            if (!thisField.equals(otherField)) {
+                return false;
             }
-            return true;
         }
-        return false;
+        return true;
     }
 
     /**
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/PersistentCollection.java b/nitrite/src/main/java/org/dizitart/no2/common/PersistentCollection.java
index 601657824..17d7fe026 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/PersistentCollection.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/PersistentCollection.java
@@ -21,21 +21,15 @@
 import org.dizitart.no2.collection.events.CollectionEventListener;
 import org.dizitart.no2.collection.events.EventAware;
 import org.dizitart.no2.collection.events.EventType;
-import org.dizitart.no2.collection.meta.MetadataAware;
+import org.dizitart.no2.common.meta.AttributesAware;
 import org.dizitart.no2.common.processors.Processor;
-import org.dizitart.no2.common.util.Iterables;
 import org.dizitart.no2.index.IndexDescriptor;
 import org.dizitart.no2.index.IndexOptions;
 import org.dizitart.no2.index.IndexType;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.store.NitriteStore;
 
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.List;
-
-import static org.dizitart.no2.common.util.ValidationUtils.containsNull;
-import static org.dizitart.no2.common.util.ValidationUtils.notNull;
 
 /**
  * The interface Persistent collection.
@@ -46,7 +40,7 @@
  * @see ObjectRepository
  * @since 1.0
  */
-public interface PersistentCollection extends EventAware, MetadataAware, AutoCloseable {
+public interface PersistentCollection extends EventAware, AttributesAware, AutoCloseable {
 
     /**
      * Adds a data processor to this collection.
@@ -56,14 +50,7 @@ public interface PersistentCollection extends EventAware, MetadataAware, Auto
     void addProcessor(Processor processor);
 
     /**
-     * Removes a data processor from this collection.
-     *
-     * @param processor the processor
-     */
-    void removeProcessor(Processor processor);
-
-    /**
-     * Creates an unique index on the {@code fields}, if not already exists.
+     * Creates a unique index on the {@code fields}, if not already exists.
      *
      * @param fields       the fields to be indexed.
      * @throws org.dizitart.no2.exceptions.IndexingException if an index already exists on the field.
@@ -98,7 +85,7 @@ default void createIndex(String... fields) {
     void createIndex(IndexOptions indexOptions, String... fields);
 
     /**
-     * Rebuilds index on the {@code field} if it exists.
+     * Rebuilds index on the {@code fields} if it exists.
      *
      * @param fields the fields to be indexed.
      * @throws org.dizitart.no2.exceptions.IndexingException if the {@code field} is not indexed.
@@ -208,32 +195,6 @@ default WriteResult update(T element) {
      */
     WriteResult update(T element, boolean insertIfAbsent);
 
-
-    /**
-     * Updates {@code elements} in the collection. Specified {@code elements} must have an id.
-     * If the {@code elements} are not found in the collection, it will be inserted only if {@code insertIfAbsent}
-     * is set to {@code true}.
-     *
-     * @param elements        the elements to update.
-     * @param insertIfAbsent if set to {@code true}, {@code elements} will be inserted if not found.
-     * @return the result of the update operation.
-     * @throws org.dizitart.no2.exceptions.ValidationException      if the {@code elements} is {@code null}.
-     * @throws org.dizitart.no2.exceptions.NotIdentifiableException if the {@code elements} does not have any id field.
-     */
-    default WriteResult update(T[] elements, boolean insertIfAbsent) {
-        notNull(elements, "a null element cannot be updated");
-        containsNull(elements, "a null element cannot be updated");
-
-        List affectedIds = new ArrayList<>();
-
-        for (T element : elements) {
-            WriteResult writeResult = update(element, insertIfAbsent);
-            affectedIds.addAll(Iterables.toList(writeResult));
-        }
-
-        return affectedIds::iterator;
-    }
-
     /**
      * Deletes the {@code element} from the collection. The {@code element} must have an id.
      *
@@ -258,7 +219,7 @@ default WriteResult update(T[] elements, boolean insertIfAbsent) {
      * Drops the collection and all of its indices.
      * 

      * Any further access to a dropped collection would result into
-     * a {@link IllegalStateException}.
+     * an exception.
      * 

      */
     void drop();
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/concurrent/ThreadPoolManager.java b/nitrite/src/main/java/org/dizitart/no2/common/concurrent/ThreadPoolManager.java
index 8ad7226b1..bf3ace37d 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/concurrent/ThreadPoolManager.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/concurrent/ThreadPoolManager.java
@@ -18,12 +18,8 @@
 
 import lombok.extern.slf4j.Slf4j;
 
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 
 import static org.dizitart.no2.common.Constants.DAEMON_THREAD_NAME;
 
@@ -40,7 +36,7 @@ public class ThreadPoolManager {
     private final static Object lock;
 
     static {
-        threadPools = new ArrayList<>();
+        threadPools = new CopyOnWriteArrayList<>();
         commonPool = workerPool();
         threadPools.add(commonPool);
         lock = new Object();
@@ -70,6 +66,20 @@ public static ExecutorService getThreadPool(int size, String threadName) {
         return threadPool;
     }
 
+    /**
+     * Creates an {@link ScheduledExecutorService} with provided size where
+     * all {@link Thread}s are daemon threads and uncaught error aware.
+     *
+     * @param size       the size
+     * @param threadName the thread name
+     * @return the {@link ScheduledExecutorService}
+     */
+    public static ScheduledExecutorService getScheduledThreadPool(int size, String threadName) {
+        ScheduledExecutorService threadPool = Executors.newScheduledThreadPool(size, threadFactory(threadName));
+        threadPools.add(threadPool);
+        return threadPool;
+    }
+
     /**
      * Returns a new {@link ErrorAwareThreadFactory} where thread name
      * will be set to the name specified.
@@ -102,29 +112,39 @@ public static Future runAsync(Runnable runnable) {
     /**
      * Shuts down all thread pools.
      */
-    public synchronized static void shutdownThreadPools() {
+    public synchronized static void shutdownAllThreadPools() {
         for (ExecutorService threadPool : threadPools) {
-            synchronized (lock) {
-                if (threadPool != null) {
-                    threadPool.shutdown();
-                }
-            }
-            try {
-                if (threadPool != null && !threadPool.awaitTermination(10, TimeUnit.SECONDS)) {
-                    synchronized (lock) {
-                        threadPool.shutdownNow();
-                    }
+            shutdownThreadPool(threadPool);
+        }
+    }
 
-                    if (!threadPool.awaitTermination(10, TimeUnit.SECONDS)) {
-                        log.error("Thread pool did not terminate");
-                    }
-                }
-            } catch (InterruptedException e) {
+    /**
+     * Shuts down a thread pool.
+     *
+     * @param threadPool the thread pool
+     */
+    public synchronized static void shutdownThreadPool(ExecutorService threadPool) {
+        synchronized (lock) {
+            if (threadPool != null) {
+                threadPool.shutdown();
+            }
+        }
+        try {
+            if (threadPool != null && !threadPool.awaitTermination(10, TimeUnit.SECONDS)) {
                 synchronized (lock) {
                     threadPool.shutdownNow();
                 }
-                Thread.currentThread().interrupt();
+
+                if (!threadPool.awaitTermination(10, TimeUnit.SECONDS)) {
+                    log.error("Thread pool did not terminate");
+                }
+            }
+        } catch (InterruptedException e) {
+            synchronized (lock) {
+                threadPool.shutdownNow();
             }
+            Thread.currentThread().interrupt();
         }
+        threadPools.remove(threadPool);
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/crypto/AESEncryptor.java b/nitrite/src/main/java/org/dizitart/no2/common/crypto/AESEncryptor.java
index 82691b7e8..f3ec0efdd 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/crypto/AESEncryptor.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/crypto/AESEncryptor.java
@@ -117,7 +117,7 @@ public String encrypt(byte[] plainText) {
             // string representation, base64, send this string to other for decryption.
             return Base64.encodeToString(cipherTextWithIvSalt, Base64.URL_SAFE);
         } catch (Exception e) {
-            throw new NitriteSecurityException("failed to encrypt data", e);
+            throw new NitriteSecurityException("Failed to encrypt data", e);
         }
     }
 
@@ -153,7 +153,7 @@ public String decrypt(String encryptedText) {
             byte[] plainText = cipher.doFinal(cipherText);
             return new String(plainText, UTF_8);
         } catch (Exception e) {
-            throw new NitriteSecurityException("failed to decrypt data", e);
+            throw new NitriteSecurityException("Failed to decrypt data", e);
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/mapper/EntityConverter.java b/nitrite/src/main/java/org/dizitart/no2/common/mapper/EntityConverter.java
new file mode 100644
index 000000000..f140cede0
--- /dev/null
+++ b/nitrite/src/main/java/org/dizitart/no2/common/mapper/EntityConverter.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.common.mapper;
+
+import org.dizitart.no2.collection.Document;
+
+/**
+ * A class that implements this interface can be used to convert
+ * entity into a database {@link Document} and back again.
+ *
+ * @since 4.0
+ * @author Anindya Chatterjee
+ * @param  the type parameter
+ */
+public interface EntityConverter {
+    /**
+     * Gets the entity type.
+     *
+     * @return the entity type
+     */
+    Class getEntityType();
+
+    /**
+     * Converts the entity to a {@link Document}.
+     *
+     * @param entity        the entity
+     * @param nitriteMapper the nitrite mapper
+     * @return the document
+     */
+    Document toDocument(T entity, NitriteMapper nitriteMapper);
+
+    /**
+     * Converts a {@link Document} to an entity of type {@link T}.
+     *
+     * @param document      the document
+     * @param nitriteMapper the nitrite mapper
+     * @return the t
+     */
+    T fromDocument(Document document, NitriteMapper nitriteMapper);
+}
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/mapper/Mappable.java b/nitrite/src/main/java/org/dizitart/no2/common/mapper/Mappable.java
deleted file mode 100644
index d16297757..000000000
--- a/nitrite/src/main/java/org/dizitart/no2/common/mapper/Mappable.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2017-2020. Nitrite author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.dizitart.no2.common.mapper;
-
-import org.dizitart.no2.collection.Document;
-
-/**
- * An object that serializes itself to a {@link Document}
- * and vice versa.
- *
- * @author Anindya Chatterjee
- * @since 2.0
- */
-public interface Mappable {
-    /**
-     * Writes the instance data to a {@link Document} and returns it.
-     *
-     * @param mapper the {@link NitriteMapper} instance used.
-     * @return the document generated.
-     */
-    Document write(NitriteMapper mapper);
-
-    /**
-     * Reads the `document` and populate all fields of this instance.
-     *
-     * @param mapper   the {@link NitriteMapper} instance used.
-     * @param document the document.
-     */
-    void read(NitriteMapper mapper, Document document);
-}
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/mapper/NitriteMapper.java b/nitrite/src/main/java/org/dizitart/no2/common/mapper/NitriteMapper.java
index 1f6e606ce..725871cd2 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/mapper/NitriteMapper.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/mapper/NitriteMapper.java
@@ -19,7 +19,7 @@
 import org.dizitart.no2.common.module.NitritePlugin;
 
 /**
- * Represents a mapper which will convert a object of one type to an object of another type.
+ * Represents a mapper which will convert an object of one type to an object of another type.
  *
  * @author Anindya Chatterjee.
  * @since 4.0
@@ -35,20 +35,4 @@ public interface NitriteMapper extends NitritePlugin {
      * @return the target
      */
      Target convert(Source source, Class type);
-
-    /**
-     * Checks if the provided type is registered as a value type.
-     *
-     * @param type the type
-     * @return the boolean
-     */
-    boolean isValueType(Class type);
-
-    /**
-     * Checks if an object is of a value type.
-     *
-     * @param object the object
-     * @return the boolean
-     */
-    boolean isValue(Object object);
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/mapper/MappableMapper.java b/nitrite/src/main/java/org/dizitart/no2/common/mapper/SimpleDocumentMapper.java
similarity index 58%
rename from nitrite/src/main/java/org/dizitart/no2/common/mapper/MappableMapper.java
rename to nitrite/src/main/java/org/dizitart/no2/common/mapper/SimpleDocumentMapper.java
index 443fb4666..c6d45b59c 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/mapper/MappableMapper.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/mapper/SimpleDocumentMapper.java
@@ -19,32 +19,83 @@
 import org.dizitart.no2.NitriteConfig;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.common.util.ObjectUtils;
 import org.dizitart.no2.exceptions.ObjectMappingException;
 
 import java.util.*;
 
 import static org.dizitart.no2.common.util.Iterables.listOf;
-import static org.dizitart.no2.common.util.ObjectUtils.newInstance;
+import static org.dizitart.no2.common.util.ValidationUtils.notNull;
 
 /**
- * A {@link NitriteMapper} based on {@link Mappable} implementation.
+ * A {@link NitriteMapper} based on {@link EntityConverter} implementation.
  *
  * @author Anindya Chatterjee.
  * @since 4.0
  */
-public class MappableMapper implements NitriteMapper {
+public class SimpleDocumentMapper implements NitriteMapper {
     private final Set> valueTypes;
+    private final Map, EntityConverter> converterRegistry;
 
     /**
-     * Instantiates a new {@link MappableMapper}.
+     * Instantiates a new {@link SimpleDocumentMapper}.
      *
      * @param valueTypes the value types
      */
-    public MappableMapper(Class... valueTypes) {
+    public SimpleDocumentMapper(Class... valueTypes) {
         this.valueTypes = new HashSet<>();
+        this.converterRegistry = new HashMap<>();
         init(listOf(valueTypes));
     }
 
+    @Override
+    @SuppressWarnings("unchecked")
+    public  Target convert(Source source, Class type) {
+        if (source == null) {
+            return null;
+        }
+
+        if (isValue(source)) {
+            return (Target) source;
+        } else {
+            if (Document.class.isAssignableFrom(type)) {
+                if (source instanceof Document) {
+                    return (Target) source;
+                } else {
+                    return (Target) convertToDocument(source);
+                }
+            } else if (source instanceof Document) {
+                return convertFromDocument((Document) source, type);
+            }
+        }
+
+        throw new ObjectMappingException("Can't convert object to type " + type
+            + ", try registering a EntityConverter for it.");
+    }
+
+    /**
+     * Adds a value type to ignore during mapping.
+     *
+     * @param valueType the value type
+     */
+    public void addValueType(Class valueType) {
+        this.valueTypes.add(valueType);
+    }
+
+    /**
+     * Registers an {@link EntityConverter}.
+     *
+     * @param entityConverter the entity converter
+     */
+    public void registerEntityConverter(EntityConverter entityConverter) {
+        notNull(entityConverter, "entityConverter cannot be null");
+        converterRegistry.put(entityConverter.getEntityType(), entityConverter);
+    }
+
+    @Override
+    public void initialize(NitriteConfig nitriteConfig) {
+    }
+
     /**
      * Converts a document to a target object of type Target.
      *
@@ -53,20 +104,19 @@ public MappableMapper(Class... valueTypes) {
      * @param type     the type
      * @return the target
      */
+    @SuppressWarnings("unchecked")
     protected  Target convertFromDocument(Document source, Class type) {
         if (source == null) {
             return null;
         }
 
-        if (Mappable.class.isAssignableFrom(type)) {
-            Target item = newInstance(type, false);
-            if (item == null) return null;
-
-            ((Mappable) item).read(this, source);
-            return item;
+        if (converterRegistry.containsKey(type)) {
+            EntityConverter serializer = (EntityConverter) converterRegistry.get(type);
+            return serializer.fromDocument(source, this);
         }
 
-        throw new ObjectMappingException("object must implements Mappable");
+        throw new ObjectMappingException("Can't convert Document to type " + type
+            + ", try registering a EntityConverter for it.");
     }
 
     /**
@@ -76,46 +126,18 @@ protected  Target convertFromDocument(Document source, Class typ
      * @param source   the source
      * @return the document
      */
-    protected  Document convertToDocument(Source source) {
-        if (source instanceof Mappable) {
-            Mappable mappable = (Mappable) source;
-            return mappable.write(this);
-        }
-
-        throw new ObjectMappingException("object must implements Mappable");
-    }
-
-    /**
-     * Adds a value type to ignore during mapping.
-     *
-     * @param valueType the value type
-     */
-    protected void addValueType(Class valueType) {
-        this.valueTypes.add(valueType);
-    }
-
-    @Override
     @SuppressWarnings("unchecked")
-    public  Target convert(Source source, Class type) {
-        if (source == null) {
-            return null;
-        }
-
-        if (isValue(source)) {
-            return (Target) source;
-        } else {
-            if (Document.class.isAssignableFrom(type)) {
-                return (Target) convertToDocument(source);
-            } else if (source instanceof Document) {
-                return convertFromDocument((Document) source, type);
-            }
+    protected  Document convertToDocument(Source source) {
+        if (converterRegistry.containsKey(source.getClass())) {
+            EntityConverter serializer = (EntityConverter) converterRegistry.get(source.getClass());
+            return serializer.toDocument(source, this);
         }
 
-        throw new ObjectMappingException("object must implements Mappable");
+        throw new ObjectMappingException("Can't convert object of type " + source.getClass().getName() +
+            " to Document, try registering a EntityConverter for it.");
     }
 
-    @Override
-    public boolean isValueType(Class type) {
+    private boolean isValueType(Class type) {
         if (type.isPrimitive() && type != void.class) return true;
         if (valueTypes.contains(type)) return true;
         for (Class valueType : valueTypes) {
@@ -124,25 +146,15 @@ public boolean isValueType(Class type) {
         return false;
     }
 
-    @Override
-    public boolean isValue(Object object) {
+    private boolean isValue(Object object) {
         return isValueType(object.getClass());
     }
 
-    @Override
-    public void initialize(NitriteConfig nitriteConfig) {
-
-    }
-
     private void init(List> valueTypes) {
-        this.valueTypes.add(Number.class);
-        this.valueTypes.add(Boolean.class);
-        this.valueTypes.add(Character.class);
-        this.valueTypes.add(String.class);
-        this.valueTypes.add(byte[].class);
+        this.valueTypes.addAll(ObjectUtils.builtInTypes());
+
         this.valueTypes.add(Enum.class);
         this.valueTypes.add(NitriteId.class);
-        this.valueTypes.add(Date.class);
 
         if (valueTypes != null && !valueTypes.isEmpty()) {
             this.valueTypes.addAll(valueTypes);
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/meta/Attributes.java b/nitrite/src/main/java/org/dizitart/no2/common/meta/Attributes.java
similarity index 71%
rename from nitrite/src/main/java/org/dizitart/no2/collection/meta/Attributes.java
rename to nitrite/src/main/java/org/dizitart/no2/common/meta/Attributes.java
index 07562d701..d338be0c3 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/meta/Attributes.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/meta/Attributes.java
@@ -1,30 +1,30 @@
 /*
- * Copyright (c) 2017-2020. Nitrite author or authors.
+ * Copyright (c) 2017-2022 Nitrite author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *       http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package org.dizitart.no2.collection.meta;
+package org.dizitart.no2.common.meta;
 
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.Data;
 
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -33,47 +33,75 @@
  * @author Anindya Chatterjee
  * @since 1.0
  */
-@EqualsAndHashCode
+@Data
 public class Attributes implements Serializable {
+    private static final long serialVersionUID = 1481284930L;
+
     /**
      * The constant CREATED_TIME.
      */
-    public static final String CREATED_TIME = "createdTime";
+    public static final String CREATED_TIME = "created_at";
+
     /**
      * The constant LAST_MODIFIED_TIME.
      */
-    public static final String LAST_MODIFIED_TIME = "lastModifiedTime";
+    public static final String LAST_MODIFIED_TIME = "last_modified_at";
+
     /**
      * The constant OWNER.
      */
     public static final String OWNER = "owner";
+
     /**
      * The constant UNIQUE_ID.
      */
     public static final String UNIQUE_ID = "uuid";
-    /**
-     * The constant LAST_SYNCED.
-     */
-    public static final String LAST_SYNCED = "lastSynced";
+
     /**
      * The constant SYNC_LOCK.
      */
-    public static final String SYNC_LOCK = "syncLock";
+    public static final String SYNC_LOCK = "sync_lock";
+
     /**
      * The constant EXPIRY_WAIT.
      */
-    public static final String EXPIRY_WAIT = "expiryWait";
+    public static final String EXPIRY_WAIT = "expiry_wait";
+
     /**
      * The constant TOMBSTONE.
      */
     public static final String TOMBSTONE = "tombstone";
+
+    /**
+     * The constant FEED_LEDGER.
+     */
+    public static final String FEED_LEDGER = "feed_ledger";
+
+    /**
+     * The constant LOCAL_COLLECTION_MARKER.
+     */
+    public static final String LOCAL_COLLECTION_MARKER = "local_collection_marker";
+
+    /**
+     * The constant REMOTE_COLLECTION_MARKER.
+     */
+    public static final String REMOTE_COLLECTION_MARKER = "remote_collection_marker";
+
+    /**
+     * The constant LOCAL_TOMBSTONE_MARKER.
+     */
+    public static final String LOCAL_TOMBSTONE_MARKER = "local_tombstone_marker";
+
+    /**
+     * The constant REMOTE_TOMBSTONE_MARKER.
+     */
+    public static final String REMOTE_TOMBSTONE_MARKER = "remote_tombstone_marker";
+
     /**
      * The constant REPLICA.
      */
     public static final String REPLICA = "replica";
-    private static final long serialVersionUID = 1481284930L;
 
-    @Getter @Setter
     private Map attributes;
 
     /**
@@ -82,7 +110,7 @@ public class Attributes implements Serializable {
     public Attributes() {
         attributes = new ConcurrentHashMap<>();
         set(CREATED_TIME, Long.toString(System.currentTimeMillis()));
-        set(UNIQUE_ID, java.util.UUID.randomUUID().toString());
+        set(UNIQUE_ID, UUID.randomUUID().toString());
     }
 
     /**
@@ -94,7 +122,7 @@ public Attributes(String collection) {
         attributes = new ConcurrentHashMap<>();
         set(OWNER, collection);
         set(CREATED_TIME, Long.toString(System.currentTimeMillis()));
-        set(UNIQUE_ID, java.util.UUID.randomUUID().toString());
+        set(UNIQUE_ID, UUID.randomUUID().toString());
     }
 
     /**
diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/meta/MetadataAware.java b/nitrite/src/main/java/org/dizitart/no2/common/meta/AttributesAware.java
similarity index 76%
rename from nitrite/src/main/java/org/dizitart/no2/collection/meta/MetadataAware.java
rename to nitrite/src/main/java/org/dizitart/no2/common/meta/AttributesAware.java
index 1f185141a..8ae092dd4 100644
--- a/nitrite/src/main/java/org/dizitart/no2/collection/meta/MetadataAware.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/meta/AttributesAware.java
@@ -1,31 +1,32 @@
 /*
- * Copyright (c) 2017-2020. Nitrite author or authors.
+ * Copyright (c) 2017-2022 Nitrite author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *       http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package org.dizitart.no2.collection.meta;
+package org.dizitart.no2.common.meta;
 
 /**
  * Interface to be implemented by database objects that wish to be
- * aware of their meta data.
+ * aware of their metadata attributes.
  *
  * @author Anindya Chatterjee.
  * @since 1.0
  */
-public interface MetadataAware {
+public interface AttributesAware {
     /**
-     * Returns the meta data attributes of an object.
+     * Returns the metadata attributes of an object.
      *
      * @return the meta data attributes.
      */
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/module/PluginManager.java b/nitrite/src/main/java/org/dizitart/no2/common/module/PluginManager.java
index f2253a789..b724975d7 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/module/PluginManager.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/module/PluginManager.java
@@ -22,7 +22,7 @@
 import org.dizitart.no2.exceptions.NitriteIOException;
 import org.dizitart.no2.exceptions.PluginException;
 import org.dizitart.no2.index.*;
-import org.dizitart.no2.common.mapper.MappableMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.store.NitriteStore;
 import org.dizitart.no2.store.memory.InMemoryStoreModule;
@@ -78,7 +78,7 @@ public void findAndLoadPlugins() {
             loadInternalPlugins();
         } catch (Exception e) {
             log.error("Error while loading internal plugins", e);
-            throw new PluginException("error while loading internal plugins", e);
+            throw new PluginException("Error while loading internal plugins", e);
         }
     }
 
@@ -90,7 +90,7 @@ public void initializePlugins() {
             initializePlugin(nitriteStore);
         } else {
             log.error("No storage engine found. Please ensure that a storage module has been loaded properly");
-            throw new NitriteIOException("no storage engine found");
+            throw new NitriteIOException("No nitrite storage engine found");
         }
 
         if (nitriteMapper != null) {
@@ -137,7 +137,7 @@ private void populatePlugins(NitritePlugin plugin) {
                 loadNitriteStore((NitriteStore) plugin);
             } else {
                 plugin.close();
-                throw new PluginException("invalid plugin loaded " + plugin);
+                throw new PluginException("Unknown plugin type: " + plugin.getClass().getName());
             }
         }
     }
@@ -145,7 +145,7 @@ private void populatePlugins(NitritePlugin plugin) {
     private void loadNitriteStore(NitriteStore nitriteStore) {
         if (this.nitriteStore != null) {
             nitriteStore.close();
-            throw new PluginException("multiple NitriteStore found");
+            throw new PluginException("Multiple nitrite store plugins found");
         }
         this.nitriteStore = nitriteStore;
     }
@@ -153,7 +153,7 @@ private void loadNitriteStore(NitriteStore nitriteStore) {
     private void loadNitriteMapper(NitriteMapper nitriteMapper) {
         if (this.nitriteMapper != null) {
             nitriteMapper.close();
-            throw new PluginException("multiple NitriteMapper found");
+            throw new PluginException("Multiple nitrite mapper plugins found");
         }
         this.nitriteMapper = nitriteMapper;
     }
@@ -161,7 +161,7 @@ private void loadNitriteMapper(NitriteMapper nitriteMapper) {
     private synchronized void loadIndexer(NitriteIndexer nitriteIndexer) {
         if (indexerMap.containsKey(nitriteIndexer.getIndexType())) {
             nitriteIndexer.close();
-            throw new PluginException("multiple Indexer found for type "
+            throw new PluginException("Multiple indexer plugins found for type: "
                 + nitriteIndexer.getIndexType());
         }
         this.indexerMap.put(nitriteIndexer.getIndexType(), nitriteIndexer);
@@ -188,7 +188,7 @@ protected void loadInternalPlugins() {
 
         if (nitriteMapper == null) {
             log.debug("Loading mappable mapper");
-            NitritePlugin plugin = new MappableMapper();
+            NitritePlugin plugin = new SimpleDocumentMapper();
             loadPlugin(plugin);
         }
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/processors/Processor.java b/nitrite/src/main/java/org/dizitart/no2/common/processors/Processor.java
index abad37dc6..7c266efe8 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/processors/Processor.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/processors/Processor.java
@@ -18,6 +18,12 @@
 package org.dizitart.no2.common.processors;
 
 import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.common.PersistentCollection;
+import org.dizitart.no2.repository.ObjectRepository;
+
+import static org.dizitart.no2.collection.UpdateOptions.updateOptions;
+import static org.dizitart.no2.common.util.DocumentUtils.createUniqueFilter;
 
 /**
  * Represents a document processor.
@@ -26,13 +32,14 @@
  * @since 4.0
  */
 public interface Processor {
+
     /**
      * Processes a document before writing it into database.
      *
      * @param document the document
      * @return the document
      */
-    Document processBeforeWrite(Document document);
+    default Document processBeforeWrite(Document document) { return document; }
 
     /**
      * Processes a document after reading from the database.
@@ -40,5 +47,28 @@ public interface Processor {
      * @param document the document
      * @return the document
      */
-    Document processAfterRead(Document document);
+    default Document processAfterRead(Document document) { return document; }
+
+    /**
+     * Processes all documents of a {@link PersistentCollection}.
+     *
+     * @param collection the collection to process
+     */
+    default void process(PersistentCollection collection) {
+        NitriteCollection nitriteCollection = null;
+        if (collection instanceof NitriteCollection) {
+            nitriteCollection = (NitriteCollection) collection;
+        } else if (collection instanceof ObjectRepository) {
+            ObjectRepository repository = (ObjectRepository) collection;
+            nitriteCollection = repository.getDocumentCollection();
+        }
+
+        if (nitriteCollection != null) {
+            for (Document document : nitriteCollection.find()) {
+                Document processed = processBeforeWrite(document);
+                nitriteCollection.update(createUniqueFilter(document), processed, updateOptions(false));
+            }
+        }
+    }
 }
+
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/processors/StringFieldEncryptionProcessor.java b/nitrite/src/main/java/org/dizitart/no2/common/processors/StringFieldEncryptionProcessor.java
index 2d702b60b..ccf6a8a46 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/processors/StringFieldEncryptionProcessor.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/processors/StringFieldEncryptionProcessor.java
@@ -88,7 +88,7 @@ public Document processBeforeWrite(Document document) {
             return copy;
         } catch (Exception e) {
             log.error("Error while processing document before write", e);
-            throw new NitriteIOException("failed to process document before write", e);
+            throw new NitriteIOException("Failed to process document before write", e);
         }
     }
 
@@ -111,7 +111,7 @@ public Document processAfterRead(Document document) {
             return copy;
         } catch (Exception e) {
             log.error("Error while processing document after read", e);
-            throw new NitriteIOException("failed to process document after read", e);
+            throw new NitriteIOException("Failed to process document after read", e);
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/BoundedDocumentStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/BoundedStream.java
similarity index 84%
rename from nitrite/src/main/java/org/dizitart/no2/common/streams/BoundedDocumentStream.java
rename to nitrite/src/main/java/org/dizitart/no2/common/streams/BoundedStream.java
index 41a978163..cd876d937 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/BoundedDocumentStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/BoundedStream.java
@@ -17,8 +17,6 @@
 
 package org.dizitart.no2.common.streams;
 
-import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.RecordStream;
 import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.exceptions.ValidationException;
@@ -33,8 +31,8 @@
  * @since 1.0
  * @author Anindya Chatterjee.
  */
-public class BoundedDocumentStream implements RecordStream> {
-    private final RecordStream> recordStream;
+public class BoundedStream implements RecordStream> {
+    private final RecordStream> recordStream;
     private final long skip;
     private final long limit;
 
@@ -45,7 +43,7 @@ public class BoundedDocumentStream implements RecordStream> recordStream) {
+    public BoundedStream(Long skip, Long limit, RecordStream> recordStream) {
         this.skip = skip;
         this.limit = limit;
 
@@ -60,8 +58,8 @@ public BoundedDocumentStream(Long skip, Long limit, RecordStream> iterator() {
-        Iterator> iterator = recordStream == null ? Collections.emptyIterator()
+    public Iterator> iterator() {
+        Iterator> iterator = recordStream == null ? Collections.emptyIterator()
             : recordStream.iterator();
         return new BoundedIterator<>(iterator, skip, limit);
     }
@@ -81,13 +79,13 @@ private static class BoundedIterator implements Iterator {
          */
         public BoundedIterator(final Iterator iterator, final long skip, final long limit) {
             if (iterator == null) {
-                throw new ValidationException("iterator must not be null");
+                throw new ValidationException("Iterator must not be null");
             }
             if (skip < 0) {
-                throw new ValidationException("skip parameter must not be negative.");
+                throw new ValidationException("skip parameter must not be negative");
             }
             if (limit < 0) {
-                throw new ValidationException("limit parameter must not be negative.");
+                throw new ValidationException("limit parameter must not be negative");
             }
 
             this.iterator = iterator;
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/UnionStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/ConcatStream.java
similarity index 92%
rename from nitrite/src/main/java/org/dizitart/no2/common/streams/UnionStream.java
rename to nitrite/src/main/java/org/dizitart/no2/common/streams/ConcatStream.java
index 1be91dc60..faf1e18c2 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/UnionStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/ConcatStream.java
@@ -25,12 +25,12 @@
 import java.util.*;
 
 /**
- * Represents an union of multiple distinct nitrite document stream.
+ * Represents an concatenation of multiple distinct nitrite document stream.
  *
  * @author Anindya Chatterjee
  * @since 4.0
  */
-public class UnionStream implements RecordStream> {
+public class ConcatStream implements RecordStream> {
     private final Collection>> streams;
 
     /**
@@ -38,7 +38,7 @@ public class UnionStream implements RecordStream> {
      *
      * @param streams the streams
      */
-    public UnionStream(Collection>> streams) {
+    public ConcatStream(Collection>> streams) {
         this.streams = streams;
     }
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentSorter.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentSorter.java
index c3b7307f6..64b7dc8ce 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentSorter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentSorter.java
@@ -22,7 +22,7 @@
 import org.dizitart.no2.common.DBNull;
 import org.dizitart.no2.common.SortOrder;
 import org.dizitart.no2.common.tuples.Pair;
-import org.dizitart.no2.exceptions.ValidationException;
+import org.dizitart.no2.exceptions.InvalidOperationException;
 
 import java.text.Collator;
 import java.util.Comparator;
@@ -75,9 +75,9 @@ public int compare(Pair pair1, Pair pa
                 } else {
 
                     // validate comparable
-                    if (value1.getClass().isArray() || value1 instanceof Iterable
-                        || value2.getClass().isArray() || value2 instanceof Iterable) {
-                        throw new ValidationException("cannot sort on an array or collection object");
+                    if (!(value1 instanceof Comparable) || !(value2 instanceof Comparable)) {
+                        throw new InvalidOperationException("Cannot compare " + value1.getClass()
+                            + " and " + value2.getClass());
                     }
 
                     // compare values
@@ -95,6 +95,7 @@ public int compare(Pair pair1, Pair pa
                     result *= -1;
                 }
 
+                // if both values are equal, continue to next sort order
                 if (result != 0) {
                     return result;
                 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java
index 66d61da9e..14aff3ddf 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java
@@ -85,7 +85,7 @@ private void validateProjection(Document projection) {
     private void validateKeyValuePair(Pair kvp) {
         if (kvp.getSecond() != null) {
             if (!(kvp.getSecond() instanceof Document)) {
-                throw new ValidationException("projection contains non-null values");
+                throw new ValidationException("Projection contains non-null values");
             } else {
                 validateProjection((Document) kvp.getSecond());
             }
@@ -127,7 +127,7 @@ public Document next() {
 
         @Override
         public void remove() {
-            throw new InvalidOperationException("remove on cursor is not supported");
+            throw new InvalidOperationException("Remove on cursor is not supported");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/FilteredStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/FilteredStream.java
index 9c20ca7d3..058f31431 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/FilteredStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/FilteredStream.java
@@ -98,7 +98,7 @@ public Pair next() {
         @Override
         public void remove() {
             if (nextPairSet) {
-                throw new InvalidOperationException("remove operation cannot be called here");
+                throw new InvalidOperationException("Remove operation cannot be called here");
             }
             iterator.remove();
         }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java
index 9bd99e034..d94368fb9 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java
@@ -27,7 +27,7 @@
 import java.util.Set;
 
 /**
- * Represents a nitrite nitrite stream backed by an index.
+ * Represents a nitrite stream backed by an index.
  *
  * @author Anindya Chatterjee
  * @since 4.0
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/JoinedDocumentStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/JoinedDocumentStream.java
index 633c37245..aeb65ea11 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/JoinedDocumentStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/JoinedDocumentStream.java
@@ -118,7 +118,7 @@ public Document next() {
 
         @Override
         public void remove() {
-            throw new InvalidOperationException("remove on a cursor is not supported");
+            throw new InvalidOperationException("Remove on a cursor is not supported");
         }
 
         private Document join(Document localDocument, DocumentCursor foreignCursor, Lookup lookup) {
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/MutatedObjectStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/MutatedObjectStream.java
similarity index 86%
rename from nitrite/src/main/java/org/dizitart/no2/repository/MutatedObjectStream.java
rename to nitrite/src/main/java/org/dizitart/no2/common/streams/MutatedObjectStream.java
index abf3845dc..856322d5c 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/MutatedObjectStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/MutatedObjectStream.java
@@ -1,20 +1,21 @@
 /*
- * Copyright (c) 2017-2020. Nitrite author or authors.
+ * Copyright (c) 2017-2022 Nitrite author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *       http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package org.dizitart.no2.repository;
+package org.dizitart.no2.common.streams;
 
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.common.RecordStream;
@@ -28,12 +29,12 @@
 /**
  * @author Anindya Chatterjee.
  */
-class MutatedObjectStream implements RecordStream {
+public class MutatedObjectStream implements RecordStream {
     private final RecordStream recordIterable;
     private final Class mutationType;
     private final NitriteMapper nitriteMapper;
 
-    MutatedObjectStream(NitriteMapper nitriteMapper,
+    public MutatedObjectStream(NitriteMapper nitriteMapper,
                         RecordStream recordIterable,
                         Class mutationType) {
         this.recordIterable = recordIterable;
@@ -73,7 +74,7 @@ public T next() {
 
         @Override
         public void remove() {
-            throw new InvalidOperationException("remove on a cursor is not supported");
+            throw new InvalidOperationException("Remove on a cursor is not supported");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/ProjectedDocumentStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/ProjectedDocumentStream.java
index 1905e35d4..fac78cbd4 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/streams/ProjectedDocumentStream.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/ProjectedDocumentStream.java
@@ -115,22 +115,23 @@ private void nextMatch() {
 
         @Override
         public void remove() {
-            throw new InvalidOperationException("remove on a cursor is not supported");
+            throw new InvalidOperationException("Remove on a cursor is not supported");
         }
 
         private Document project(Document original) {
             if (projection == null) return original;
-            Document result = original.clone();
+            Document newDoc = Document.createDocument();
 
-            for (Pair pair : original) {
-                if (!projection.containsKey(pair.getFirst())) {
-                    result.remove(pair.getFirst());
+            for (String field : projection.getFields()) {
+                if (original.containsField(field)) {
+                    Object value = original.get(field);
+                    newDoc.put(field, value);
                 }
             }
 
             // process the result
-            result = processorChain.processAfterRead(result);
-            return result;
+            newDoc = processorChain.processAfterRead(newDoc);
+            return newDoc;
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/Base64.java b/nitrite/src/main/java/org/dizitart/no2/common/util/Base64.java
index 4e4363e48..19989791f 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/Base64.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/Base64.java
@@ -92,7 +92,7 @@ public static byte[] decode(byte[] input, int offset, int len, int flags) {
         // (It could contain less if it contains whitespace, etc.)
         Decoder decoder = new Decoder(flags, new byte[len*3/4]);
         if (!decoder.process(input, offset, len)) {
-            throw new IllegalArgumentException("bad base-64");
+            throw new IllegalArgumentException("Bad base-64 input");
         }
         // Maybe we got lucky and allocated exactly enough output space.
         if (decoder.op == decoder.output.length) {
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/Comparables.java b/nitrite/src/main/java/org/dizitart/no2/common/util/Comparables.java
index 0c13bcd20..3cf45de25 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/Comparables.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/Comparables.java
@@ -23,7 +23,9 @@ public static int compare(Comparable first, Comparable second) {
             Number number2 = (Number) second;
             int result = Numbers.compare(number1, number2);
             if (!first.getClass().equals(second.getClass())) {
-                if (result == 0) return 1;
+                if (result == 0) {
+                    return first.toString().compareTo(second.toString()) < 0 ? -1 : 1;
+                }
             }
             return result;
         }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/DocumentUtils.java b/nitrite/src/main/java/org/dizitart/no2/common/util/DocumentUtils.java
index 9c92db1ab..904cb81c5 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/DocumentUtils.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/DocumentUtils.java
@@ -70,10 +70,7 @@ public static Filter createUniqueFilter(Document document) {
      * @return the document
      */
     public static  Document skeletonDocument(NitriteMapper nitriteMapper, Class type) {
-        if (nitriteMapper.isValueType(type)) {
-            return Document.createDocument();
-        }
-        T dummy = newInstance(type, true);
+        T dummy = newInstance(type, true, nitriteMapper);
         Document document = nitriteMapper.convert(dummy, Document.class);
         return removeValues(document);
     }
@@ -106,13 +103,14 @@ public static FieldValues getValues(Document document, Fields fields) {
 
     private static Document removeValues(Document document) {
         if (document == null) return null;
+        Document newDoc = Document.createDocument();
         for (Pair entry : document) {
             if (entry.getSecond() instanceof Document) {
-                document.put(entry.getFirst(), removeValues((Document) entry.getSecond()));
+                newDoc.put(entry.getFirst(), removeValues((Document) entry.getSecond()));
             } else {
-                document.put(entry.getFirst(), null);
+                newDoc.put(entry.getFirst(), null);
             }
         }
-        return document;
+        return newDoc;
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/IndexUtils.java b/nitrite/src/main/java/org/dizitart/no2/common/util/IndexUtils.java
index bfb9413b1..a304200d1 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/IndexUtils.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/IndexUtils.java
@@ -41,7 +41,7 @@ public static String deriveIndexMapName(IndexDescriptor descriptor) {
             INTERNAL_NAME_SEPARATOR +
             descriptor.getCollectionName() +
             INTERNAL_NAME_SEPARATOR +
-            descriptor.getIndexFields().getEncodedName() +
+            descriptor.getFields().getEncodedName() +
             INTERNAL_NAME_SEPARATOR +
             descriptor.getIndexType();
     }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/Iterables.java b/nitrite/src/main/java/org/dizitart/no2/common/util/Iterables.java
index 55b980d91..27f745b54 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/Iterables.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/Iterables.java
@@ -16,8 +16,6 @@
 
 package org.dizitart.no2.common.util;
 
-import org.dizitart.no2.common.UnknownType;
-
 import java.lang.reflect.Array;
 import java.util.*;
 
@@ -138,15 +136,6 @@ public static long size(Iterable iterable) {
         return count;
     }
 
-    public static Class getElementType(Iterable iterable) {
-        if (iterable == null) return UnknownType.class;
-        Iterator iterator = iterable.iterator();
-        if (iterator.hasNext()) {
-            return iterator.next().getClass();
-        }
-        return UnknownType.class;
-    }
-
     @SuppressWarnings({"unchecked", "rawtypes"})
     static Object[] toArray(Iterable iterable) {
         if (iterable instanceof Collection) {
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/Numbers.java b/nitrite/src/main/java/org/dizitart/no2/common/util/Numbers.java
index afcfcc875..fb5cee5ad 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/Numbers.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/Numbers.java
@@ -48,27 +48,6 @@ public static int compare(Number x, Number y) {
         }
     }
 
-    public static Object castNumber(Object value, Class type) {
-        if (value instanceof Number && Number.class.isAssignableFrom(type)) {
-            Number number = (Number) value;
-            if (type.equals(Short.class) || type.equals(short.class)) {
-                return number.shortValue();
-            } else if (type.equals(Byte.class) || type.equals(byte.class)) {
-                return number.byteValue();
-            } else if (type.equals(Double.class) || type.equals(double.class)) {
-                return number.doubleValue();
-            } else if (type.equals(Float.class) || type.equals(float.class)) {
-                return number.floatValue();
-            } else if (type.equals(Integer.class) || type.equals(int.class)) {
-                return number.intValue();
-            } else if (type.equals(Long.class) || type.equals(long.class)) {
-                return number.longValue();
-            }
-        }
-        throw new ValidationException("cannot cast number of type " + value.getClass().getName()
-            + " to " + type.getName());
-    }
-
     private static boolean isSpecial(Number number) {
         boolean specialDouble = number instanceof Double
             && (Double.isNaN((Double) number) || Double.isInfinite((Double) number));
@@ -91,7 +70,7 @@ private static BigDecimal toBigDecimal(Number number) {
         try {
             return new BigDecimal(number.toString());
         } catch (NumberFormatException e) {
-            throw new ValidationException("the given number (\"" + number + "\" of class "
+            throw new ValidationException("The given number (\"" + number + "\" of class "
                 + number.getClass().getName() + ") does not have a parsable string representation", e);
         }
     }
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/ObjectUtils.java b/nitrite/src/main/java/org/dizitart/no2/common/util/ObjectUtils.java
index e61c139a4..1385617d3 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/ObjectUtils.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/ObjectUtils.java
@@ -17,21 +17,25 @@
 package org.dizitart.no2.common.util;
 
 import lombok.extern.slf4j.Slf4j;
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.exceptions.NitriteIOException;
 import org.dizitart.no2.exceptions.ObjectMappingException;
 import org.dizitart.no2.exceptions.ValidationException;
+import org.dizitart.no2.repository.EntityDecorator;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.annotations.Entity;
-import org.objenesis.Objenesis;
-import org.objenesis.ObjenesisSerializer;
-import org.objenesis.ObjenesisStd;
-import org.objenesis.instantiator.ObjectInstantiator;
 
 import java.io.*;
 import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.URI;
+import java.net.URL;
 import java.util.*;
+import java.util.regex.Pattern;
 
 import static org.dizitart.no2.common.Constants.KEY_OBJ_SEPARATOR;
 import static org.dizitart.no2.common.util.Iterables.toArray;
@@ -46,8 +50,6 @@
 @Slf4j
 public class ObjectUtils {
     private static final Map, Class> PRIMITIVE_TO_WRAPPER_TYPE;
-    private static final Objenesis stdObjenesis = new ObjenesisStd(true);
-    private static final Objenesis serializerObjenesis = new ObjenesisSerializer(true);
 
     static {
         Map, Class> primToWrap = new LinkedHashMap<>();
@@ -92,6 +94,14 @@ public static String findRepositoryName(String entityName, String key) {
         }
     }
 
+    public static  String findRepositoryNameByDecorator(EntityDecorator entityDecorator, String key) {
+        String entityName = entityDecorator.getEntityName();
+        if (entityName.contains(KEY_OBJ_SEPARATOR)) {
+            throw new ValidationException(entityName + " is not a valid entity name");
+        }
+        return findRepositoryName(entityName, key);
+    }
+
     /**
      * Gets the key name of a keyed-{@link ObjectRepository}
      *
@@ -179,19 +189,15 @@ public static boolean deepEquals(Object o1, Object o2) {
             // generic check
             return o1.equals(o2);
         }
-
-        // none of the type check passes so they are not of compatible type
     }
 
-    @SuppressWarnings({"unchecked", "rawtypes"})
-    public static  T newInstance(Class type, boolean createSkeleton) {
+    public static  T newInstance(Class type, boolean createSkeleton, NitriteMapper nitriteMapper) {
         try {
-            if (type.isPrimitive() || type.isArray() || type == String.class) {
+            if (type.isPrimitive() || type.isArray() || type == String.class || isBuiltInValueType(type)) {
                 return defaultValue(type);
             }
 
-            ObjectInstantiator instantiator = getInstantiatorOf(type);
-            T item = (T) instantiator.newInstance();
+            T item = nitriteMapper.convert(Document.createDocument(), type);
 
             if (createSkeleton) {
                 Field[] fields = type.getDeclaredFields();
@@ -201,17 +207,8 @@ public static  T newInstance(Class type, boolean createSkeleton) {
                         if (!Modifier.isStatic(field.getModifiers())) {
                             field.setAccessible(true);
 
-                            // remove final modifier
-                            try {
-                                Field modifiersField = Field.class.getDeclaredField("modifiers");
-                                modifiersField.setAccessible(true);
-                                modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
-                            } catch (NoSuchFieldException e) {
-                                // ignore in case of java 12+
-                            }
-
                             if (isSkeletonRequired(type, field.getType())) {
-                                field.set(item, newInstance(field.getType(), true));
+                                field.set(item, newInstance(field.getType(), true, nitriteMapper));
                             } else {
                                 field.set(item, defaultValue(field.getType()));
                             }
@@ -222,18 +219,49 @@ public static  T newInstance(Class type, boolean createSkeleton) {
 
             return item;
         } catch (Throwable e) {
-            throw new ObjectMappingException("failed to instantiate type " + type.getName(), e);
+            throw new ObjectMappingException("Failed to instantiate type " + type.getName(), e);
         }
     }
 
-    public static boolean isValueType(Class retType) {
+    public static boolean isBuiltInValueType(Class retType) {
         if (retType.isPrimitive() && retType != void.class) return true;
         if (Number.class.isAssignableFrom(retType)) return true;
-        if (Boolean.class == retType) return true;
-        if (Character.class == retType) return true;
-        if (String.class == retType) return true;
         if (byte[].class.isAssignableFrom(retType)) return true;
-        return Enum.class.isAssignableFrom(retType);
+        if (Enum.class.isAssignableFrom(retType)) return true;
+        if (builtInTypes().contains(retType)) return true;
+        for (Class builtInType : builtInTypes()) {
+            if (builtInType.isAssignableFrom(retType)) return true;
+        }
+        return false;
+    }
+
+    public static List> builtInTypes() {
+        return Iterables.listOf(
+            byte[].class,
+            Number.class,
+            Byte.class,
+            Short.class,
+            Integer.class,
+            Long.class,
+            Float.class,
+            Double.class,
+            BigDecimal.class,
+            BigInteger.class,
+            Boolean.class,
+            Character.class,
+            String.class,
+            Date.class,
+            URL.class,
+            URI.class,
+            Currency.class,
+            Calendar.class,
+            StringBuffer.class,
+            StringBuilder.class,
+            Locale.class,
+            Void.class,
+            UUID.class,
+            Pattern.class
+        );
     }
 
     public static boolean isCompatibleTypes(Class type1, Class type2) {
@@ -285,7 +313,7 @@ public static  T deepCopy(T oldObj) {
                 return (T) ois.readObject();
             }
         } catch (IOException | ClassNotFoundException e) {
-            throw new NitriteIOException("error while deep copying object", e);
+            throw new NitriteIOException("Error while deep copying object", e);
         }
     }
 
@@ -294,14 +322,6 @@ private static Class toWrapperType(Class type) {
         return (wrapped == null) ? type : wrapped;
     }
 
-    private static  ObjectInstantiator getInstantiatorOf(Class type) {
-        if (Serializable.class.isAssignableFrom(type)) {
-            return serializerObjenesis.getInstantiatorOf(type);
-        } else {
-            return stdObjenesis.getInstantiatorOf(type);
-        }
-    }
-
     private static  boolean isSkeletonRequired(Class
 enclosingType, Class fieldType) {
         String fieldTypePackage = getPackageName(fieldType);
         String enclosingTypePackage = getPackageName(enclosingType);
@@ -324,7 +344,7 @@ private static  String getPackageName(Class clazz) {
     }
 
     @SuppressWarnings("unchecked")
-    private static  T defaultValue(Class type) {
+    public static  T defaultValue(Class type) {
         if (type.isPrimitive()) {
             switch (type.getName()) {
                 case "boolean":
@@ -351,7 +371,7 @@ private static  T defaultValue(Class type) {
         }
 
         if (type == String.class) {
-            return (T) "";
+            return (T) null;
         }
 
         return null;
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/SpatialKey.java b/nitrite/src/main/java/org/dizitart/no2/common/util/SpatialKey.java
new file mode 100644
index 000000000..da9d8e1b4
--- /dev/null
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/SpatialKey.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.common.util;
+
+import java.util.Arrays;
+
+/*
+ * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (https://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+public class SpatialKey {
+    private final long id;
+    private final float[] minMax;
+
+    public SpatialKey(long id, float... minMax) {
+        this.id = id;
+        this.minMax = minMax;
+    }
+
+    public float min(int dim) {
+        return minMax[dim + dim];
+    }
+
+    public void setMin(int dim, float x) {
+        minMax[dim + dim] = x;
+    }
+
+    public float max(int dim) {
+        return minMax[dim + dim + 1];
+    }
+
+    public void setMax(int dim, float x) {
+        minMax[dim + dim + 1] = x;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public boolean isNull() {
+        return minMax.length == 0;
+    }
+
+    @Override
+    public int hashCode() {
+        return (int) ((id >>> 32) ^ id);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) {
+            return true;
+        } else if (!(other instanceof SpatialKey)) {
+            return false;
+        }
+        SpatialKey o = (SpatialKey) other;
+        if (id != o.id) {
+            return false;
+        }
+        return equalsIgnoringId(o);
+    }
+
+    public boolean equalsIgnoringId(SpatialKey o) {
+        return Arrays.equals(minMax, o.minMax);
+    }
+}
diff --git a/nitrite/src/main/java/org/dizitart/no2/common/util/ValidationUtils.java b/nitrite/src/main/java/org/dizitart/no2/common/util/ValidationUtils.java
index 60afea147..f6e3c7ceb 100644
--- a/nitrite/src/main/java/org/dizitart/no2/common/util/ValidationUtils.java
+++ b/nitrite/src/main/java/org/dizitart/no2/common/util/ValidationUtils.java
@@ -16,13 +16,16 @@
 
 package org.dizitart.no2.common.util;
 
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.exceptions.IndexingException;
 import org.dizitart.no2.exceptions.InvalidOperationException;
 import org.dizitart.no2.exceptions.ValidationException;
 
+import java.lang.reflect.Modifier;
 import java.util.Collection;
 
-import static org.dizitart.no2.common.util.ObjectUtils.convertToObjectArray;
+import static org.dizitart.no2.common.util.ObjectUtils.*;
 import static org.dizitart.no2.common.util.StringUtils.isNullOrEmpty;
 
 /**
@@ -161,33 +164,73 @@ public static void validateFilterIterableField(Iterable fieldValue, String fi
         }
     }
 
+    public static void validateProjectionType(Class type, NitriteMapper nitriteMapper) {
+        Object value;
+        try {
+            value = newInstance(type, false, nitriteMapper);
+        } catch (Exception e) {
+            throw new ValidationException("Invalid projection type", e);
+        }
+
+        if (value == null) {
+            throw new ValidationException("Invalid projection type");
+        }
+
+        Document document = nitriteMapper.convert(value, Document.class);
+        if (document == null || document.size() == 0) {
+            throw new ValidationException("Cannot project to empty type " + type);
+        }
+    }
+
+    public static void validateRepositoryType(Class type, NitriteMapper nitriteMapper) {
+        Object value;
+        try {
+            if (type.isInterface() || (Modifier.isAbstract(type.getModifiers()) && !isBuiltInValueType(type))) {
+                // defer validation during insertion
+                return;
+            }
+
+            value = newInstance(type, false, nitriteMapper);
+            if (value == null) {
+                throw new ValidationException("Cannot create new instance of type " + type);
+            }
+
+            Document document = nitriteMapper.convert(value, Document.class);
+            if (document == null || document.size() == 0) {
+                throw new ValidationException("Cannot convert to document from type " + type);
+            }
+        } catch (Exception e) {
+            throw new ValidationException("Invalid repository type", e);
+        }
+    }
+
     private static void validateArrayIndexItem(Object value, String field) {
         if (value instanceof Iterable || value.getClass().isArray()) {
-            throw new InvalidOperationException("nested array index on iterable field " + field + " is not supported");
+            throw new InvalidOperationException("Nested iterables are not supported");
         }
 
         if (!(value instanceof Comparable)) {
-            throw new IndexingException("cannot index on an array field containing non comparable values " + field);
+            throw new IndexingException("Each value in the iterable field " + field + " must implement Comparable");
         }
     }
 
     private static void validateStringArrayItem(Object value, String field) {
         if (!(value instanceof String) && (value instanceof Iterable || value.getClass().isArray())) {
-            throw new InvalidOperationException("nested array index on iterable field " + field + " is not supported");
+            throw new InvalidOperationException("Nested iterables are not supported");
         }
 
         if (!(value instanceof String)) {
-            throw new IndexingException("cannot index on an array field containing non string values " + field);
+            throw new IndexingException("Each value in the iterable field " + field + " must be a string");
         }
     }
 
     private static void validateArrayFilterItem(Object value, String field) {
         if (value instanceof Iterable || value.getClass().isArray()) {
-            throw new InvalidOperationException("nested array is not supported");
+            throw new InvalidOperationException("Nested array is not supported");
         }
 
         if (!(value instanceof Comparable)) {
-            throw new IndexingException("cannot filter using non comparable values " + field);
+            throw new IndexingException("Cannot filter using non comparable values " + field);
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/exceptions/MigrationException.java b/nitrite/src/main/java/org/dizitart/no2/exceptions/MigrationException.java
index 2980689dd..d74aa030f 100644
--- a/nitrite/src/main/java/org/dizitart/no2/exceptions/MigrationException.java
+++ b/nitrite/src/main/java/org/dizitart/no2/exceptions/MigrationException.java
@@ -15,4 +15,14 @@ public class MigrationException extends NitriteException {
     public MigrationException(String errorMessage) {
         super(errorMessage);
     }
+
+    /**
+     * Instantiates a new Migration exception.
+     *
+     * @param errorMessage the error message
+     * @param cause the cause
+     */
+    public MigrationException(String errorMessage, Throwable cause) {
+        super(errorMessage, cause);
+    }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/AndFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/AndFilter.java
index 9e646111b..4a345de20 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/AndFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/AndFilter.java
@@ -41,18 +41,19 @@ public class AndFilter extends LogicalFilter {
 
         for (int i = 1; i < filters.length; i++) {
             if (filters[i] instanceof TextFilter) {
-                throw new FilterException("text filter must be the first filter in AND operation");
+                throw new FilterException("Text filter must be the first filter in AND operation");
             }
         }
     }
 
     @Override
     public boolean apply(Pair element) {
-        boolean result = true;
         for (Filter filter : getFilters()) {
-            result = result && filter.apply(element);
+            if (!filter.apply(element)) {
+                return false;
+            }
         }
-        return result;
+        return true;
     }
 
     @Override
@@ -61,11 +62,10 @@ public String toString() {
         stringBuilder.append("(");
         for (int i = 0; i < getFilters().size(); i++) {
             Filter filter = getFilters().get(i);
-            if (i == 0) {
-                stringBuilder.append(filter.toString());
-            } else {
-                stringBuilder.append(" && ").append(filter.toString());
+            if (i > 0) {
+                stringBuilder.append(" && ");
             }
+            stringBuilder.append(filter.toString());
         }
         stringBuilder.append(")");
         return stringBuilder.toString();
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/BetweenFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/BetweenFilter.java
index a8693caf6..903b32121 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/BetweenFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/BetweenFilter.java
@@ -17,7 +17,7 @@
 package org.dizitart.no2.filters;
 
 import lombok.Data;
-import org.dizitart.no2.exceptions.ValidationException;
+import org.dizitart.no2.exceptions.FilterException;
 
 /**
  * @author Anindya Chatterjee
@@ -50,11 +50,11 @@ private static  Filter getLhs(String field, Bound bound) {
 
     private static  void validateBound(Bound bound) {
         if (bound == null) {
-            throw new ValidationException("bound cannot be null");
+            throw new FilterException("Bound cannot be null");
         }
 
         if (!(bound.upperBound instanceof Comparable) || !(bound.lowerBound instanceof Comparable)) {
-            throw new ValidationException("upper bound or lower bound value must be comparable");
+            throw new FilterException("Upper bound or lower bound value must be comparable");
         }
     }
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/ComparableFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/ComparableFilter.java
index e2d7c64fc..4946f95d5 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/ComparableFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/ComparableFilter.java
@@ -54,7 +54,7 @@ public Comparable getComparable() {
     }
 
     /**
-     * Apply this filter on an nitrite index.
+     * Apply this filter on a nitrite index.
      *
      * @param indexMap the index scanner
      * @return the object
@@ -73,7 +73,7 @@ protected void processIndexValue(Object value,
                                      List, Object>> subMap,
                                      List nitriteIds) {
         if (value instanceof List) {
-            // if its is list then add it directly to nitrite ids
+            // if it is list then add it directly to nitrite ids
             List result = (List) value;
             nitriteIds.addAll(result);
         }
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/ElementMatchFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/ElementMatchFilter.java
index 842d303fd..64709e8b9 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/ElementMatchFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/ElementMatchFilter.java
@@ -47,11 +47,11 @@ class ElementMatchFilter extends NitriteFilter {
     @SuppressWarnings({"rawtypes", "unchecked"})
     public boolean apply(Pair element) {
         if (elementFilter instanceof ElementMatchFilter) {
-            throw new FilterException("nested elemMatch filter is not supported");
+            throw new FilterException("Nested elemMatch filter is not supported");
         }
 
         if (elementFilter instanceof TextFilter) {
-            throw new FilterException("text filter is not supported in elemMatch filter");
+            throw new FilterException("Text filter is not supported in elemMatch filter");
         }
 
         Document document = element.getSecond();
@@ -128,7 +128,7 @@ private boolean matchElement(Object item, Filter filter) {
         } else if (filter instanceof RegexFilter) {
             return matchRegex(item, filter);
         } else {
-            throw new FilterException("filter " + filter.getClass().getSimpleName() +
+            throw new FilterException("Filter " + filter.getClass().getSimpleName() +
                 " is not a supported in elemMatch");
         }
     }
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/EqualsFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/EqualsFilter.java
index c290f4896..20ad2a85a 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/EqualsFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/EqualsFilter.java
@@ -44,6 +44,10 @@ public boolean apply(Pair element) {
     @Override
     public List applyOnIndex(IndexMap indexMap) {
         Object value = indexMap.get((Comparable) getValue());
+        if (value == null) {
+            return new ArrayList<>();
+        }
+
         if (value instanceof List) {
             return ((List) value);
         }
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/FieldBasedFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/FieldBasedFilter.java
index 9112d515e..91ae7969b 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/FieldBasedFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/FieldBasedFilter.java
@@ -20,7 +20,6 @@
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import org.dizitart.no2.exceptions.ValidationException;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 import static org.dizitart.no2.common.util.ValidationUtils.notEmpty;
@@ -55,7 +54,7 @@ protected FieldBasedFilter(String field, Object value) {
     }
 
     /**
-     * Gets the value fo the filter.
+     * Gets the value of the filter.
      *
      * @return the value
      */
@@ -67,7 +66,7 @@ public Object getValue() {
         if (getObjectFilter()) {
             NitriteMapper nitriteMapper = getNitriteConfig().nitriteMapper();
             validateSearchTerm(nitriteMapper, field, value);
-            if (nitriteMapper.isValue(value)) {
+            if (value instanceof Comparable) {
                 value = nitriteMapper.convert(value, Comparable.class);
             }
         }
@@ -79,11 +78,5 @@ public Object getValue() {
     protected void validateSearchTerm(NitriteMapper nitriteMapper, String field, Object value) {
         notNull(field, "field cannot be null");
         notEmpty(field, "field cannot be empty");
-
-        if (value != null) {
-            if (!nitriteMapper.isValue(value) && !(value instanceof Comparable)) {
-                throw new ValidationException("search term is not comparable " + value);
-            }
-        }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/Filter.java b/nitrite/src/main/java/org/dizitart/no2/filters/Filter.java
index bb884ff63..b816c6209 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/Filter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/Filter.java
@@ -64,9 +64,9 @@ static Filter byId(NitriteId nitriteId) {
      * @return the filter
      */
     static Filter and(Filter... filters) {
-        notEmpty(filters, "at least two filters must be specified");
+        notEmpty(filters, "At least two filters must be specified");
         if (filters.length < 2) {
-            throw new FilterException("at least two filters must be specified");
+            throw new FilterException("At least two filters must be specified");
         }
 
         return new AndFilter(filters);
@@ -79,9 +79,9 @@ static Filter and(Filter... filters) {
      * @return the filter
      */
     static Filter or(Filter... filters) {
-        notEmpty(filters, "at least two filters must be specified");
+        notEmpty(filters, "At least two filters must be specified");
         if (filters.length < 2) {
-            throw new FilterException("at least two filters must be specified");
+            throw new FilterException("At least two filters must be specified");
         }
 
         return new OrFilter(filters);
@@ -97,7 +97,7 @@ static Filter or(Filter... filters) {
 
     /**
      * Creates a not filter which performs a logical NOT operation on a filter and selects
-     * the documents that *_do not_* satisfy the criteria. This also includes documents
+     * the documents that do not satisfy the criteria. This also includes documents
      * that do not contain the value.
      * 

      *
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/FluentFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/FluentFilter.java
index 98d7d00bb..f222d7e05 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/FluentFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/FluentFilter.java
@@ -17,7 +17,7 @@
 package org.dizitart.no2.filters;
 
 /**
- * A fluent api for the {@link Filter}.
+ * A fluent api for the {@link NitriteFilter}.
  *
  * @author Anindya Chatterjee.
  * @since 4.0
@@ -34,7 +34,7 @@ private FluentFilter() {
     }
 
     /**
-     * Where clause for fluent filter.
+     * Where clause for fluent filter api.
      *
      * @param field the field
      * @return the fluent filter
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/GreaterEqualFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/GreaterEqualFilter.java
index f3c8374bb..93b5c4e2b 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/GreaterEqualFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/GreaterEqualFilter.java
@@ -60,7 +60,7 @@ public boolean apply(Pair element) {
     @SuppressWarnings({"unchecked", "rawtypes"})
     public List applyOnIndex(IndexMap indexMap) {
         Comparable comparable = getComparable();
-        List, Object>> subMap = new ArrayList<>();
+        List, Object>> subMaps = new ArrayList<>();
 
         // maintain the find sorting order
         List nitriteIds = new ArrayList<>();
@@ -70,14 +70,14 @@ public List applyOnIndex(IndexMap indexMap) {
             // get the starting value, it can be a navigable-map (compound index)
             // or list (single field index)
             Object value = indexMap.get(ceilingKey);
-            processIndexValue(value, subMap, nitriteIds);
+            processIndexValue(value, subMaps, nitriteIds);
 
             ceilingKey = indexMap.higherKey(ceilingKey);
         }
 
-        if (!subMap.isEmpty()) {
+        if (!subMaps.isEmpty()) {
             // if sub-map is populated then filtering on compound index, return sub-map
-            return subMap;
+            return subMaps;
         } else {
             // else it is filtering on either single field index,
             // or it is a terminal filter on compound index, return only nitrite-ids
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/GreaterThanFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/GreaterThanFilter.java
index a41b71a92..089b34133 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/GreaterThanFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/GreaterThanFilter.java
@@ -61,7 +61,7 @@ public boolean apply(Pair element) {
     @SuppressWarnings({"unchecked", "rawtypes"})
     public List applyOnIndex(IndexMap indexMap) {
         Comparable comparable = getComparable();
-        List, Object>> subMap = new ArrayList<>();
+        List, Object>> subMaps = new ArrayList<>();
         List nitriteIds = new ArrayList<>();
 
         Comparable ceilingKey = indexMap.higherKey(comparable);
@@ -69,14 +69,14 @@ public List applyOnIndex(IndexMap indexMap) {
             // get the starting value, it can be a navigable-map (compound index)
             // or list (single field index)
             Object value = indexMap.get(ceilingKey);
-            processIndexValue(value, subMap, nitriteIds);
+            processIndexValue(value, subMaps, nitriteIds);
 
             ceilingKey = indexMap.higherKey(ceilingKey);
         }
 
-        if (!subMap.isEmpty()) {
+        if (!subMaps.isEmpty()) {
             // if sub-map is populated then filtering on compound index, return sub-map
-            return subMap;
+            return subMaps;
         } else {
             // else it is filtering on either single field index,
             // or it is a terminal filter on compound index, return only nitrite-ids
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/IndexOnlyFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/IndexOnlyFilter.java
index 5eac4a099..9e5bdc2e8 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/IndexOnlyFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/IndexOnlyFilter.java
@@ -35,6 +35,11 @@ public IndexOnlyFilter(String field, Object value) {
         super(field, value);
     }
 
+    /**
+     * Gets the supported index type for this filter.
+     *
+     * @return the supported index type
+     */
     public abstract String supportedIndexType();
 
     /**
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/IndexScanFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/IndexScanFilter.java
index 03282e782..a7c167673 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/IndexScanFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/IndexScanFilter.java
@@ -50,6 +50,6 @@ public IndexScanFilter(Collection filters) {
 
     @Override
     public boolean apply(Pair element) {
-        throw new InvalidOperationException("index scan filter cannot be applied on collection");
+        throw new InvalidOperationException("Index scan filter cannot be applied on collection");
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/LesserEqualFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/LesserEqualFilter.java
index 0a87b0dbc..c248d6294 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/LesserEqualFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/LesserEqualFilter.java
@@ -63,14 +63,14 @@ public List applyOnIndex(IndexMap indexMap) {
         List, Object>> subMap = new ArrayList<>();
         List nitriteIds = new ArrayList<>();
 
-        Comparable ceilingKey = indexMap.floorKey(comparable);
-        while (ceilingKey != null) {
+        Comparable floorKey = indexMap.floorKey(comparable);
+        while (floorKey != null) {
             // get the starting value, it can be a navigable-map (compound index)
             // or list (single field index)
-            Object value = indexMap.get(ceilingKey);
+            Object value = indexMap.get(floorKey);
             processIndexValue(value, subMap, nitriteIds);
 
-            ceilingKey = indexMap.lowerKey(ceilingKey);
+            floorKey = indexMap.lowerKey(floorKey);
         }
 
         if (!subMap.isEmpty()) {
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/LesserThanFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/LesserThanFilter.java
index e522dbca8..93e35a839 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/LesserThanFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/LesserThanFilter.java
@@ -63,14 +63,14 @@ public List applyOnIndex(IndexMap indexMap) {
         List, Object>> subMap = new ArrayList<>();
         List nitriteIds = new ArrayList<>();
 
-        Comparable ceilingKey = indexMap.lowerKey(comparable);
-        while (ceilingKey != null) {
+        Comparable floorKey = indexMap.lowerKey(comparable);
+        while (floorKey != null) {
             // get the starting value, it can be a navigable-map (compound index)
             // or list (single field index)
-            Object value = indexMap.get(ceilingKey);
+            Object value = indexMap.get(floorKey);
             processIndexValue(value, subMap, nitriteIds);
 
-            ceilingKey = indexMap.lowerKey(ceilingKey);
+            floorKey = indexMap.lowerKey(floorKey);
         }
 
         if (!subMap.isEmpty()) {
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/OrFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/OrFilter.java
index 7ad22a6e1..96ecfbb89 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/OrFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/OrFilter.java
@@ -40,11 +40,12 @@ public class OrFilter extends LogicalFilter {
 
     @Override
     public boolean apply(Pair element) {
-        boolean result = false;
         for (Filter filter : getFilters()) {
-            result = result || filter.apply(element);
+            if (filter.apply(element)) {
+                return true;
+            }
         }
-        return result;
+        return false;
     }
 
     @Override
@@ -53,11 +54,10 @@ public String toString() {
         stringBuilder.append("(");
         for (int i = 0; i < getFilters().size(); i++) {
             Filter filter = getFilters().get(i);
-            if (i == 0) {
-                stringBuilder.append(filter.toString());
-            } else {
-                stringBuilder.append(" || ").append(filter.toString());
+            if (i > 0) {
+                stringBuilder.append(" || ");
             }
+            stringBuilder.append(filter.toString());
         }
         stringBuilder.append(")");
         return stringBuilder.toString();
diff --git a/nitrite/src/main/java/org/dizitart/no2/filters/TextFilter.java b/nitrite/src/main/java/org/dizitart/no2/filters/TextFilter.java
index 154e797f7..b1d02f4de 100644
--- a/nitrite/src/main/java/org/dizitart/no2/filters/TextFilter.java
+++ b/nitrite/src/main/java/org/dizitart/no2/filters/TextFilter.java
@@ -58,7 +58,7 @@ public boolean apply(Pair element) {
         Object docValue = element.getSecond().get(getField());
 
         if (!(docValue instanceof String)) {
-            throw new FilterException("text filter can not be applied on non string field " + getField());
+            throw new FilterException("Text filter can not be applied on non string field " + getField());
         }
 
         String docString = (String) docValue;
@@ -76,12 +76,12 @@ public String toString() {
     }
 
     /**
-     * Apply on index linked hash set.
+     * Apply this filter on text index.
      *
      * @param indexMap the index map
      * @return the linked hash set
      */
-    public LinkedHashSet applyOnIndex(NitriteMap> indexMap) {
+    public LinkedHashSet applyOnTextIndex(NitriteMap> indexMap) {
         notNull(getField(), "field cannot be null");
         notNull(getStringValue(), "search term cannot be null");
         String searchString = getStringValue();
@@ -117,12 +117,13 @@ private LinkedHashSet searchExactByIndex(NitriteMap>
 
     private LinkedHashSet searchByWildCard(NitriteMap> indexMap, String searchString) {
         if (searchString.contentEquals("*")) {
-            throw new FilterException("* is not a valid search string");
+            throw new FilterException("* is not a valid search term");
         }
 
         StringTokenizer stringTokenizer = stringTokenizer(searchString);
         if (stringTokenizer.countTokens() > 1) {
-            throw new FilterException("multiple words with wildcard is not supported");
+            throw new FilterException("Wild card search can not be applied on " +
+                "multiple words");
         }
 
         if (searchString.startsWith("*") && !searchString.endsWith("*")) {
@@ -138,7 +139,7 @@ private LinkedHashSet searchByWildCard(NitriteMap> in
     @SuppressWarnings("unchecked")
     private LinkedHashSet searchByLeadingWildCard(NitriteMap> indexMap, String searchString) {
         if (searchString.equalsIgnoreCase("*")) {
-            throw new FilterException("invalid search term '*'");
+            throw new FilterException("* is not a valid search term");
         }
 
         LinkedHashSet idSet = new LinkedHashSet<>();
@@ -156,7 +157,7 @@ private LinkedHashSet searchByLeadingWildCard(NitriteMap searchByTrailingWildCard(NitriteMap> indexMap, String searchString) {
         if (searchString.equalsIgnoreCase("*")) {
-            throw new FilterException("invalid search term '*'");
+            throw new FilterException("* is not a valid search term");
         }
 
         LinkedHashSet idSet = new LinkedHashSet<>();
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java b/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java
index 0064753be..4d2d24320 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java
@@ -21,6 +21,7 @@
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.FieldValues;
 import org.dizitart.no2.common.Fields;
+import org.dizitart.no2.exceptions.IndexingException;
 
 import java.util.LinkedHashSet;
 import java.util.Map;
@@ -85,6 +86,10 @@ public void dropIndex(IndexDescriptor indexDescriptor, NitriteConfig nitriteConf
     }
 
     private NitriteIndex findNitriteIndex(IndexDescriptor indexDescriptor, NitriteConfig nitriteConfig) {
+        if (indexDescriptor == null) {
+            throw new IndexingException("Index descriptor cannot be null");
+        }
+
         if (indexRegistry.containsKey(indexDescriptor)) {
             return indexRegistry.get(indexDescriptor);
         }
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/CompoundIndex.java b/nitrite/src/main/java/org/dizitart/no2/index/CompoundIndex.java
index beec17621..74d1b979a 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/CompoundIndex.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/CompoundIndex.java
@@ -21,6 +21,7 @@
 import org.dizitart.no2.collection.FindPlan;
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.DBNull;
+import org.dizitart.no2.common.DBValue;
 import org.dizitart.no2.common.FieldValues;
 import org.dizitart.no2.common.Fields;
 import org.dizitart.no2.common.tuples.Pair;
@@ -169,26 +170,26 @@ private void removeIndexElement(NitriteMap> in
     }
 
     @SuppressWarnings({"rawtypes", "unchecked"})
-    private void populateSubMap(NavigableMap subMap, FieldValues fieldValues, int startIndex) {
-        if (startIndex >= fieldValues.getValues().size()) return;
+    private void populateSubMap(NavigableMap subMap, FieldValues fieldValues, int depth) {
+        if (depth >= fieldValues.getValues().size()) return;
 
-        Pair pair = fieldValues.getValues().get(startIndex);
+        Pair pair = fieldValues.getValues().get(depth);
         Object value = pair.getSecond();
         DBValue dbValue;
         if (value == null) {
             dbValue = DBNull.getInstance();
         } else {
             if (Iterable.class.isAssignableFrom(value.getClass()) || value.getClass().isArray()) {
-                throw new IndexingException("compound multikey index is supported on the first field of the index only");
+                throw new IndexingException("Compound multikey index is supported on the first field of the index only");
             }
 
             if (!(value instanceof Comparable)) {
-                throw new IndexingException(value + " is not comparable");
+                throw new IndexingException(value + " is not a comparable type");
             }
             dbValue = new DBValue((Comparable) value);
         }
 
-        if (startIndex == fieldValues.getValues().size() - 1) {
+        if (depth == fieldValues.getValues().size() - 1) {
             // terminal field
             List nitriteIds = (List) subMap.get(dbValue);
             nitriteIds = addNitriteIds(nitriteIds, fieldValues);
@@ -202,13 +203,13 @@ private void populateSubMap(NavigableMap subMap, FieldValues fieldValues, int st
             }
 
             subMap.put(dbValue, subMap2);
-            populateSubMap(subMap2, fieldValues, startIndex + 1);
+            populateSubMap(subMap2, fieldValues, depth + 1);
         }
     }
 
     @SuppressWarnings({"rawtypes", "unchecked"})
-    private void deleteFromSubMap(NavigableMap subMap, FieldValues fieldValues, int startIndex) {
-        Pair pair = fieldValues.getValues().get(startIndex);
+    private void deleteFromSubMap(NavigableMap subMap, FieldValues fieldValues, int depth) {
+        Pair pair = fieldValues.getValues().get(depth);
         Object value = pair.getSecond();
         DBValue dbValue;
         if (value == null) {
@@ -220,7 +221,7 @@ private void deleteFromSubMap(NavigableMap subMap, FieldValues fieldValues, int
             dbValue = new DBValue((Comparable) value);
         }
 
-        if (startIndex == fieldValues.getValues().size() - 1) {
+        if (depth == fieldValues.getValues().size() - 1) {
             // terminal field
             List nitriteIds = (List) subMap.get(dbValue);
             nitriteIds = removeNitriteIds(nitriteIds, fieldValues);
@@ -236,7 +237,7 @@ private void deleteFromSubMap(NavigableMap subMap, FieldValues fieldValues, int
                 return;
             }
 
-            deleteFromSubMap(subMap2, fieldValues, startIndex + 1);
+            deleteFromSubMap(subMap2, fieldValues, depth + 1);
             subMap.put(dbValue, subMap2);
         }
     }
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/IndexDescriptor.java b/nitrite/src/main/java/org/dizitart/no2/index/IndexDescriptor.java
index 1b97b6b7a..0198249cc 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/IndexDescriptor.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/IndexDescriptor.java
@@ -54,11 +54,10 @@ public class IndexDescriptor implements Comparable, Serializabl
     /**
      * Gets the target fields for the index.
      *
-     * @param indexFields fields in the index
      * @return the target fields.
      */
     @Getter
-    private Fields indexFields;
+    private Fields fields;
 
     /**
      * Gets the collection name.
@@ -83,7 +82,7 @@ public IndexDescriptor(String indexType, Fields fields, String collectionName) {
         notEmpty(collectionName, "collectionName cannot be empty");
 
         this.indexType = indexType;
-        this.indexFields = fields;
+        this.fields = fields;
         this.collectionName = collectionName;
     }
 
@@ -91,7 +90,7 @@ public IndexDescriptor(String indexType, Fields fields, String collectionName) {
     public int compareTo(IndexDescriptor other) {
         if (other == null) return 1;
 
-        // compound index have highest cardinality
+        // compound index have the highest cardinality
         if (this.isCompoundIndex() && !other.isCompoundIndex()) return 1;
 
         // unique index has the next highest cardinality
@@ -100,25 +99,25 @@ public int compareTo(IndexDescriptor other) {
         // for two unique indices, the one with encompassing higher
         // number of fields has the higher cardinality
         if (this.isUniqueIndex()) {
-            return this.indexFields.compareTo(other.indexFields);
+            return this.fields.compareTo(other.fields);
         }
 
         // for two non-unique indices, the one with encompassing higher
         // number of fields has the higher cardinality
         if (!other.isUniqueIndex()) {
-            return this.indexFields.compareTo(other.indexFields);
+            return this.fields.compareTo(other.fields);
         }
 
         return -1;
     }
 
     /**
-     * Is compound index boolean.
+     * Indicates if this descriptor is for a compound index.
      *
      * @return the boolean
      */
     public boolean isCompoundIndex() {
-        return indexFields.getFieldNames().size() > 1;
+        return fields.getFieldNames().size() > 1;
     }
 
     private boolean isUniqueIndex() {
@@ -127,13 +126,13 @@ private boolean isUniqueIndex() {
 
     private void writeObject(ObjectOutputStream stream) throws IOException {
         stream.writeUTF(indexType);
-        stream.writeObject(indexFields);
+        stream.writeObject(fields);
         stream.writeUTF(collectionName);
     }
 
     private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException {
         indexType = stream.readUTF();
-        indexFields = (Fields) stream.readObject();
+        fields = (Fields) stream.readObject();
         collectionName = stream.readUTF();
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/IndexMap.java b/nitrite/src/main/java/org/dizitart/no2/index/IndexMap.java
index 8eecb4f4f..828c7ca21 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/IndexMap.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/IndexMap.java
@@ -21,6 +21,7 @@
 import lombok.Setter;
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.DBNull;
+import org.dizitart.no2.common.DBValue;
 import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.store.NitriteMap;
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/IndexOptions.java b/nitrite/src/main/java/org/dizitart/no2/index/IndexOptions.java
index 75bf1d6b6..64b0c4424 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/IndexOptions.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/IndexOptions.java
@@ -40,8 +40,8 @@ public class IndexOptions {
     private String indexType;
 
     /**
-     * Creates an {@link IndexOptions} with the specified `indexType`. Index creation
-     * will be synchronous with this option.
+     * Creates an {@link IndexOptions} with the specified indexType.
+     * Index creation will be synchronous with this option.
      *
      * @param indexType the type of index to be created.
      * @return a new synchronous index creation option.
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/IndexScanner.java b/nitrite/src/main/java/org/dizitart/no2/index/IndexScanner.java
index 0236b8395..5f0c81763 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/IndexScanner.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/IndexScanner.java
@@ -18,6 +18,7 @@
 package org.dizitart.no2.index;
 
 import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.common.DBValue;
 import org.dizitart.no2.exceptions.FilterException;
 import org.dizitart.no2.filters.ComparableFilter;
 
@@ -97,10 +98,10 @@ public LinkedHashSet doScan(List filters, Map terminalResult = indexMap.getTerminalNitriteIds();
             nitriteIds.addAll(terminalResult);
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java b/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java
index 8881058e2..836a0007a 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java
@@ -23,7 +23,8 @@
 import org.dizitart.no2.exceptions.UniqueConstraintException;
 import org.dizitart.no2.exceptions.ValidationException;
 
-import java.util.*;
+import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 import static org.dizitart.no2.common.util.ValidationUtils.validateArrayIndexField;
@@ -91,10 +92,8 @@ default void validateIndexField(Object value, String field) {
             validateIterableIndexField((Iterable) value, field);
         } else if (value.getClass().isArray()) {
             validateArrayIndexField(value, field);
-        } else {
-            if (!(value instanceof Comparable)) {
-                throw new ValidationException(value + " is not comparable");
-            }
+        } else if (!(value instanceof Comparable)) {
+            throw new ValidationException("Index field " + field + " must be a comparable type");
         }
     }
 
@@ -110,10 +109,9 @@ default List addNitriteIds(List nitriteIds, FieldValues fi
             nitriteIds = new CopyOnWriteArrayList<>();
         }
 
-        if (isUnique() && nitriteIds.size() == 1
-            && !nitriteIds.contains(fieldValues.getNitriteId())) {
+        if (isUnique() && nitriteIds.size() == 1) {
             // if key is already exists for unique type, throw error
-            throw new UniqueConstraintException("unique key constraint violation for " + fieldValues.getFields());
+            throw new UniqueConstraintException("Unique key constraint violation for " + fieldValues.getFields());
         }
 
         // index always are in ascending format
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/NitriteTextIndexer.java b/nitrite/src/main/java/org/dizitart/no2/index/NitriteTextIndexer.java
index 683cc0f73..e01b20707 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/NitriteTextIndexer.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/NitriteTextIndexer.java
@@ -69,7 +69,7 @@ public String getIndexType() {
     @Override
     public void validateIndex(Fields fields) {
         if (fields.getFieldNames().size() > 1) {
-            throw new IndexingException("text index can only be created on a single field");
+            throw new IndexingException("Text index can only be created on a single field");
         }
     }
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java b/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java
index 592caad86..2d6ae64ab 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java
@@ -21,10 +21,10 @@
 import org.dizitart.no2.collection.FindPlan;
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.DBNull;
+import org.dizitart.no2.common.DBValue;
 import org.dizitart.no2.common.FieldValues;
 import org.dizitart.no2.common.Fields;
 import org.dizitart.no2.filters.ComparableFilter;
-import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.store.NitriteMap;
 import org.dizitart.no2.store.NitriteStore;
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/TextIndex.java b/nitrite/src/main/java/org/dizitart/no2/index/TextIndex.java
index 3c6a4aba4..dcb344364 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/TextIndex.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/TextIndex.java
@@ -97,7 +97,8 @@ public void write(FieldValues fieldValues) {
                 addIndexElement(indexMap, fieldValues, (String) item);
             }
         } else {
-            throw new IndexingException("string data is expected");
+            throw new IndexingException("Index field " + firstField
+                + " must be a String, String[] or Iterable");
         }
     }
 
@@ -129,7 +130,8 @@ public void remove(FieldValues fieldValues) {
                 removeIndexElement(indexMap, fieldValues, (String) item);
             }
         } else {
-            throw new IndexingException("string data is expected");
+            throw new IndexingException("Index field " + firstField
+                + " must be a String, String[] or Iterable");
         }
     }
 
@@ -150,9 +152,9 @@ public LinkedHashSet findNitriteIds(FindPlan findPlan) {
         if (filters.size() == 1 && filters.get(0) instanceof TextFilter) {
             TextFilter textFilter = (TextFilter) filters.get(0);
             textFilter.setTextTokenizer(textTokenizer);
-            return textFilter.applyOnIndex(indexMap);
+            return textFilter.applyOnTextIndex(indexMap);
         }
-        throw new FilterException("invalid filter found for full-text index");
+        throw new FilterException("Text index only supports a single TextFilter");
     }
 
     private NitriteMap> findIndexMap() {
@@ -192,29 +194,8 @@ private void removeIndexElement(NitriteMap> indexMap, FieldValue
         }
     }
 
-    private Set decompose(Object fieldValue) {
-        Set result = new HashSet<>();
-        if (fieldValue == null) {
-            result.add(null);
-        } else if (fieldValue instanceof String) {
-            result.add((String) fieldValue);
-        } else if (fieldValue instanceof Iterable) {
-            Iterable iterable = (Iterable) fieldValue;
-            for (Object item : iterable) {
-                result.addAll(decompose(item));
-            }
-        } else if (fieldValue.getClass().isArray()) {
-            Object[] array = convertToObjectArray(fieldValue);
-            for (Object item : array) {
-                result.addAll(decompose(item));
-            }
-        }
-
-        Set words = new HashSet<>();
-        for (String item : result) {
-            words.addAll(textTokenizer.tokenize(item));
-        }
-
-        return words;
+    private Set decompose(String fieldValue) {
+        if (fieldValue == null) return new HashSet<>();
+        return textTokenizer.tokenize(fieldValue);
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/fulltext/BaseTextTokenizer.java b/nitrite/src/main/java/org/dizitart/no2/index/fulltext/BaseTextTokenizer.java
index ffedc3096..9d224176e 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/fulltext/BaseTextTokenizer.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/fulltext/BaseTextTokenizer.java
@@ -50,18 +50,18 @@ public Set tokenize(String text) {
     }
 
     /**
-     * Converts a `word` into all lower case and checks if it
-     * is a known stop word. If it is, then the `word` will be
+     * Converts a word into all lower case and checks if it
+     * is a known stop word. If it is, then the word will be
      * discarded and will not be considered as a valid token.
      *
      * @param word the word
      * @return the tokenized word in all upper case.
      */
     protected String convertWord(String word) {
-        word = word.toLowerCase();
-        if (stopWords().contains(word)) {
+        String convertedWord = word.toLowerCase();
+        if (stopWords().contains(convertedWord)) {
             return null;
         }
-        return word;
+        return convertedWord;
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/fulltext/TextTokenizer.java b/nitrite/src/main/java/org/dizitart/no2/index/fulltext/TextTokenizer.java
index 5b7876d5f..9c9b1e07e 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/fulltext/TextTokenizer.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/fulltext/TextTokenizer.java
@@ -35,7 +35,7 @@ public interface TextTokenizer {
     Languages getLanguage();
 
     /**
-     * Tokenize a `text` and discards all stop-words from it.
+     * Tokenize a text and discards all stop-words from it.
      *
      * @param text the text to tokenize
      * @return the set of tokens.
diff --git a/nitrite/src/main/java/org/dizitart/no2/index/fulltext/UniversalTextTokenizer.java b/nitrite/src/main/java/org/dizitart/no2/index/fulltext/UniversalTextTokenizer.java
index 975d03288..c9e302cd4 100644
--- a/nitrite/src/main/java/org/dizitart/no2/index/fulltext/UniversalTextTokenizer.java
+++ b/nitrite/src/main/java/org/dizitart/no2/index/fulltext/UniversalTextTokenizer.java
@@ -236,6 +236,8 @@ private void loadLanguage(Languages... languages) {
                 case Zulu:
                     registerLanguage(new Zulu());
                     break;
+                default:
+                    break;
             }
         }
     }
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/CollectionInstruction.java b/nitrite/src/main/java/org/dizitart/no2/migration/CollectionInstruction.java
index edc12009c..ebce6123f 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/CollectionInstruction.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/CollectionInstruction.java
@@ -24,6 +24,7 @@ default CollectionInstruction rename(String name) {
         addStep(migrationStep);
         final CollectionInstruction parent = this;
 
+        // new instruction set for new collection
         return new CollectionInstruction() {
             @Override
             public String collectionName() {
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/DatabaseInstruction.java b/nitrite/src/main/java/org/dizitart/no2/migration/DatabaseInstruction.java
index 3f6f919e4..cd2bf594d 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/DatabaseInstruction.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/DatabaseInstruction.java
@@ -3,6 +3,9 @@
 import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.common.tuples.Triplet;
 import org.dizitart.no2.common.util.SecureString;
+import org.dizitart.no2.repository.EntityDecorator;
+
+import static org.dizitart.no2.common.util.ObjectUtils.getEntityName;
 
 /**
  * Represents a migration instruction set for the nitrite database.
@@ -13,7 +16,7 @@
 public interface DatabaseInstruction extends Instruction {
 
     /**
-     * Adds an instruction to set an user authentication to the database.
+     * Adds an instruction to set a user authentication to the database.
      *
      * @param username the username
      * @param password the password
@@ -64,28 +67,48 @@ default DatabaseInstruction dropCollection(String collectionName) {
      * @return the database instruction
      */
     default DatabaseInstruction dropRepository(Class type) {
-        return dropRepository(type.getName());
+        return dropRepository(type, null);
+    }
+
+    /**
+     * Adds an instruction to drop a keyed {@link org.dizitart.no2.repository.ObjectRepository} from the database.
+     *
+     * @param type the type
+     * @param key  the key
+     * @return the database instruction
+     */
+    default DatabaseInstruction dropRepository(Class type, String key) {
+        return dropRepository(getEntityName(type), key);
     }
 
     /**
      * Adds an instruction to drop an {@link org.dizitart.no2.repository.ObjectRepository} from the database.
      *
-     * @param typeName the type name
+     * @param entityDecorator the entityDecorator
      * @return the database instruction
      */
-    default DatabaseInstruction dropRepository(String typeName) {
-        return dropRepository(typeName, null);
+    default DatabaseInstruction dropRepository(EntityDecorator entityDecorator) {
+        return dropRepository(entityDecorator, null);
     }
 
     /**
-     * Adds an instruction to drop a keyed {@link org.dizitart.no2.repository.ObjectRepository} from the database.
+     * Adds an instruction to drop an {@link org.dizitart.no2.repository.ObjectRepository} from the database.
      *
-     * @param type the type
-     * @param key  the key
+     * @param entityDecorator the entityDecorator
      * @return the database instruction
      */
-    default DatabaseInstruction dropRepository(Class type, String key) {
-        return dropRepository(type.getName(), key);
+    default DatabaseInstruction dropRepository(EntityDecorator entityDecorator, String key) {
+        return dropRepository(entityDecorator.getEntityName(), key);
+    }
+
+    /**
+     * Adds an instruction to drop an {@link org.dizitart.no2.repository.ObjectRepository} from the database.
+     *
+     * @param typeName the type name
+     * @return the database instruction
+     */
+    default DatabaseInstruction dropRepository(String typeName) {
+        return dropRepository(typeName, null);
     }
 
     /**
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/Instructions.java b/nitrite/src/main/java/org/dizitart/no2/migration/InstructionSet.java
similarity index 67%
rename from nitrite/src/main/java/org/dizitart/no2/migration/Instructions.java
rename to nitrite/src/main/java/org/dizitart/no2/migration/InstructionSet.java
index 5f98c267e..5848490dc 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/Instructions.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/InstructionSet.java
@@ -1,5 +1,7 @@
 package org.dizitart.no2.migration;
 
+import org.dizitart.no2.repository.EntityDecorator;
+
 import static org.dizitart.no2.common.util.ObjectUtils.getEntityName;
 
 /**
@@ -8,7 +10,7 @@
  * @author Anindya Chatterjee
  * @since 4.0
  */
-public interface Instructions {
+public interface InstructionSet {
     /**
      * Creates a {@link DatabaseInstruction}.
      *
@@ -19,32 +21,53 @@ public interface Instructions {
     /**
      * Creates a {@link RepositoryInstruction}.
      *
-     * @param typeName the type name
+     * @param type the type
      * @return the repository instruction
      */
-    default RepositoryInstruction forRepository(String typeName) {
-        return forRepository(typeName, null);
+    default RepositoryInstruction forRepository(Class type) {
+        return forRepository(type, null);
     }
 
     /**
      * Creates a {@link RepositoryInstruction}.
      *
      * @param type the type
+     * @param key  the key
      * @return the repository instruction
      */
-    default RepositoryInstruction forRepository(Class type) {
-        return forRepository(getEntityName(type), null);
+    default RepositoryInstruction forRepository(Class type, String key) {
+        return forRepository(getEntityName(type), key);
     }
 
     /**
      * Creates a {@link RepositoryInstruction}.
      *
-     * @param type the type
+     * @param entityDecorator the entityDecorator
+     * @return the repository instruction
+     */
+    default RepositoryInstruction forRepository(EntityDecorator entityDecorator) {
+        return forRepository(entityDecorator, null);
+    }
+
+    /**
+     * Creates a {@link RepositoryInstruction}.
+     *
+     * @param entityDecorator the entityDecorator
      * @param key  the key
      * @return the repository instruction
      */
-    default RepositoryInstruction forRepository(Class type, String key) {
-        return forRepository(getEntityName(type), key);
+    default RepositoryInstruction forRepository(EntityDecorator entityDecorator, String key) {
+        return forRepository(entityDecorator.getEntityName(), key);
+    }
+
+    /**
+     * Creates a {@link RepositoryInstruction}.
+     *
+     * @param typeName the type name
+     * @return the repository instruction
+     */
+    default RepositoryInstruction forRepository(String typeName) {
+        return forRepository(typeName, null);
     }
 
     /**
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/Migration.java b/nitrite/src/main/java/org/dizitart/no2/migration/Migration.java
index 92af007be..a75354881 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/Migration.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/Migration.java
@@ -15,31 +15,31 @@ public abstract class Migration {
     private final Queue migrationSteps;
 
     @Getter
-    private final Integer startVersion;
+    private final Integer fromVersion;
 
     @Getter
-    private final Integer endVersion;
+    private final Integer toVersion;
 
     private boolean executed = false;
 
     /**
      * Instantiates a new {@link Migration}.
      *
-     * @param startVersion the start version
-     * @param endVersion   the end version
+     * @param fromVersion the start version
+     * @param toVersion   the end version
      */
-    public Migration(Integer startVersion, Integer endVersion) {
-        this.startVersion = startVersion;
-        this.endVersion = endVersion;
+    public Migration(Integer fromVersion, Integer toVersion) {
+        this.fromVersion = fromVersion;
+        this.toVersion = toVersion;
         this.migrationSteps = new LinkedList<>();
     }
 
     /**
      * Migrates the database using the instructions.
      *
-     * @param instructions the instructions
+     * @param instructionSet the instructions
      */
-    public abstract void migrate(Instructions instructions);
+    public abstract void migrate(InstructionSet instructionSet);
 
     /**
      * Returns the {@link MigrationStep}s as a queue for execution.
@@ -54,7 +54,7 @@ public Queue steps() {
     }
 
     private void execute() {
-        NitriteInstructions instruction = new NitriteInstructions(migrationSteps);
+        NitriteInstructionSet instruction = new NitriteInstructionSet(migrationSteps);
         migrate(instruction);
         this.executed = true;
     }
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/MigrationManager.java b/nitrite/src/main/java/org/dizitart/no2/migration/MigrationManager.java
index 61aef06d2..5190ef3d7 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/MigrationManager.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/MigrationManager.java
@@ -59,10 +59,10 @@ public void doMigrate() {
                 try {
                     database.close();
                 } catch (Exception e) {
-                    throw new NitriteIOException("failed to close the database", e);
+                    throw new NitriteIOException("Failed to close the database", e);
                 }
 
-                throw new MigrationException("schema version mismatch, as no migration path found from version "
+                throw new MigrationException("Schema version mismatch, no migration path found from version "
                     + storeMetadata.getSchemaVersion() + " to " + nitriteConfig.getSchemaVersion());
             }
 
@@ -82,12 +82,13 @@ private boolean isMigrationNeeded() {
         Integer incomingVersion = nitriteConfig.getSchemaVersion();
 
         if (existingVersion == null) {
-            throw new MigrationException("corrupted database, no version information found");
+            throw new MigrationException("Corrupted database, no version information found");
         }
 
         if (incomingVersion == null) {
-            throw new MigrationException("invalid version provided");
+            throw new MigrationException("Invalid version provided");
         }
+
         return !existingVersion.equals(incomingVersion);
     }
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/NitriteInstructions.java b/nitrite/src/main/java/org/dizitart/no2/migration/NitriteInstructionSet.java
similarity index 80%
rename from nitrite/src/main/java/org/dizitart/no2/migration/NitriteInstructions.java
rename to nitrite/src/main/java/org/dizitart/no2/migration/NitriteInstructionSet.java
index 3dade87ec..01f56f3ee 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/NitriteInstructions.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/NitriteInstructionSet.java
@@ -6,16 +6,16 @@
 import java.util.Queue;
 
 /**
- * Default implementation of {@link Instructions}.
+ * Default implementation of {@link InstructionSet}.
  *
  * @author Anindya Chatterjee
  * @since 4.0
  */
-class NitriteInstructions implements Instructions {
+class NitriteInstructionSet implements InstructionSet {
     @Getter(AccessLevel.PACKAGE)
     private final Queue migrationSteps;
 
-    NitriteInstructions(Queue migrationSteps) {
+    NitriteInstructionSet(Queue migrationSteps) {
         this.migrationSteps = migrationSteps;
     }
 
@@ -25,11 +25,11 @@ public DatabaseInstruction forDatabase() {
     }
 
     @Override
-    public RepositoryInstruction forRepository(String typeName, String key) {
+    public RepositoryInstruction forRepository(String entityName, String key) {
         return new RepositoryInstruction() {
             @Override
             public String entityName() {
-                return typeName;
+                return entityName;
             }
 
             @Override
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/RepositoryInstruction.java b/nitrite/src/main/java/org/dizitart/no2/migration/RepositoryInstruction.java
index 7dd160cac..c24f304c0 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/RepositoryInstruction.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/RepositoryInstruction.java
@@ -27,8 +27,10 @@ default RepositoryInstruction renameRepository(String entityName, String key) {
         migrationStep.setInstructionType(InstructionType.RenameRepository);
         migrationStep.setArguments(new Quartet<>(entityName(), key(), entityName, key));
         addStep(migrationStep);
+
         final RepositoryInstruction parent = this;
 
+        // new instruction set for new repository
         return new RepositoryInstruction() {
             @Override
             public String entityName() {
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/commands/Rename.java b/nitrite/src/main/java/org/dizitart/no2/migration/commands/Rename.java
index 923aa4205..5a2bcb47e 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/commands/Rename.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/commands/Rename.java
@@ -39,7 +39,7 @@ public void execute(Nitrite nitrite) {
             try (IndexManager indexManager = new IndexManager(oldName, nitrite.getConfig())) {
                 Collection indexEntries = indexManager.getIndexDescriptors();
                 for (IndexDescriptor indexDescriptor : indexEntries) {
-                    Fields field = indexDescriptor.getIndexFields();
+                    Fields field = indexDescriptor.getFields();
                     String indexType = indexDescriptor.getIndexType();
                     newOperations.createIndex(field, indexType);
                 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/migration/commands/RenameField.java b/nitrite/src/main/java/org/dizitart/no2/migration/commands/RenameField.java
index 6f9f92ec8..564a4878b 100644
--- a/nitrite/src/main/java/org/dizitart/no2/migration/commands/RenameField.java
+++ b/nitrite/src/main/java/org/dizitart/no2/migration/commands/RenameField.java
@@ -47,9 +47,9 @@ public void execute(Nitrite nitrite) {
                 for (IndexDescriptor matchingIndexDescriptor : matchingIndexDescriptors) {
                     String indexType = matchingIndexDescriptor.getIndexType();
 
-                    Fields oldIndexFields = matchingIndexDescriptor.getIndexFields();
+                    Fields oldIndexFields = matchingIndexDescriptor.getFields();
                     Fields newIndexFields = getNewIndexFields(oldIndexFields, oldName, newName);
-                    operations.dropIndex(matchingIndexDescriptor.getIndexFields());
+                    operations.dropIndex(matchingIndexDescriptor.getFields());
                     operations.createIndex(newIndexFields, indexType);
                 }
             }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/AnnotationScanner.java b/nitrite/src/main/java/org/dizitart/no2/repository/AnnotationScanner.java
index 2fb209025..03f85f4ea 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/AnnotationScanner.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/AnnotationScanner.java
@@ -49,12 +49,12 @@ public AnnotationScanner(Class type, NitriteCollection collection, NitriteMap
         this.collection = collection;
         this.reflector = new Reflector();
         this.indices = new HashSet<>();
-        this.indexValidator = new IndexValidator(reflector);
+        this.indexValidator = new IndexValidator();
     }
 
     public void createIndices() {
         for (Index index : indices) {
-            String[] fields = index.value();
+            String[] fields = index.fields();
             if (!collection.hasIndex(fields)) {
                 collection.createIndex(indexOptions(index.type()), fields);
             }
@@ -63,14 +63,14 @@ public void createIndices() {
 
     public void createIdIndex() {
         if (objectIdField != null) {
-            String[] fieldNames = objectIdField.getFieldNames(nitriteMapper);
+            String[] fieldNames = objectIdField.getEmbeddedFieldNames();
             if (!collection.hasIndex(fieldNames)) {
                 collection.createIndex(fieldNames);
             }
         }
     }
 
-    public void scanIndices() {
+    public void performScan() {
         // populate from @Indices
         scanIndicesAnnotation();
 
@@ -137,35 +137,24 @@ private void scanIdAnnotation() {
             if (field.isAnnotationPresent(Id.class)) {
                 Id id = field.getAnnotation(Id.class);
                 String fieldName = StringUtils.isNullOrEmpty(id.fieldName()) ? field.getName() : id.fieldName();
-                indexValidator.validate(field.getType(), fieldName, nitriteMapper);
+                indexValidator.validateId(id, field.getType(), fieldName, nitriteMapper);
                 if (alreadyIdFound) {
-                    throw new NotIdentifiableException("multiple id fields found for the type");
+                    throw new NotIdentifiableException("Multiple id fields found for the type");
                 } else {
                     alreadyIdFound = true;
                     objectIdField = new ObjectIdField();
                     objectIdField.setField(field);
                     objectIdField.setIdFieldName(fieldName);
-                    objectIdField.setEmbedded(isEmbeddedId(field));
+                    objectIdField.setEmbedded(id.embeddedFields().length > 0);
+                    objectIdField.setFieldNames(id.embeddedFields());
                 }
             }
         }
     }
 
-    private boolean isEmbeddedId(Field field) {
-        List fields = reflector.getAllFields(field.getType());
-        if (fields.size() == 0) return false;
-
-        for (Field f : fields) {
-            if (f.isAnnotationPresent(Embedded.class)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     private void populateIndex(List indexList) {
         for (Index index : indexList) {
-            String[] names = index.value();
+            String[] names = index.fields();
             List entityFields = new ArrayList<>();
 
             for (String name : names) {
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/Cursor.java b/nitrite/src/main/java/org/dizitart/no2/repository/Cursor.java
index 5d6049923..50aaa3402 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/Cursor.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/Cursor.java
@@ -17,13 +17,11 @@
 package org.dizitart.no2.repository;
 
 import org.dizitart.no2.collection.FindPlan;
-import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.Lookup;
 import org.dizitart.no2.common.RecordStream;
 
 /**
- * A collection of {@link NitriteId}s of the database records,
- * as a result of a find operation.
+ * An interface to iterate over {@link ObjectRepository#find()} results.
  *
  * @author Anindya Chatterjee
  * @since 1.0
@@ -37,7 +35,7 @@ public interface Cursor extends RecordStream {
     FindPlan getFindPlan();
 
     /**
-     * Projects the result of one type into an {@link Iterable} of other type.
+     * Projects the result of one type into an {@link Iterable} of another type.
      *
      * @param 
            the type of the target objects.
      * @param projectionType the projection type.
@@ -48,8 +46,8 @@ public interface Cursor extends RecordStream {
     /**
      * Performs a left outer join with a foreign cursor with the specified lookup parameters.
      * 

-     * It performs an equality match on the localString to the foreignString from the objects of the foreign cursor.
-     * If an input object does not contain the localString, the join treats the field as having a value of null
+     * It performs an equality match on the localField to the foreignField from the objects of the foreign cursor.
+     * If an input object does not contain the localField, the join treats the field as having a value of null
      * for matching purposes.
      *
      * @param      the type of the foreign object.
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/DefaultObjectRepository.java b/nitrite/src/main/java/org/dizitart/no2/repository/DefaultObjectRepository.java
index f6bebc643..b007dfeb6 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/DefaultObjectRepository.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/DefaultObjectRepository.java
@@ -20,14 +20,15 @@
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.FindOptions;
 import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.collection.UpdateOptions;
 import org.dizitart.no2.collection.events.CollectionEventListener;
-import org.dizitart.no2.collection.meta.Attributes;
 import org.dizitart.no2.common.WriteResult;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.meta.Attributes;
+import org.dizitart.no2.common.processors.Processor;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.index.IndexDescriptor;
 import org.dizitart.no2.index.IndexOptions;
-import org.dizitart.no2.common.mapper.NitriteMapper;
-import org.dizitart.no2.common.processors.Processor;
 import org.dizitart.no2.store.NitriteStore;
 
 import java.util.Collection;
@@ -42,7 +43,8 @@
 class DefaultObjectRepository implements ObjectRepository {
     private final NitriteCollection collection;
     private final NitriteConfig nitriteConfig;
-    private final Class type;
+    private Class type;
+    private EntityDecorator entityDecorator;
     private RepositoryOperations operations;
 
     DefaultObjectRepository(Class type,
@@ -54,18 +56,21 @@ class DefaultObjectRepository implements ObjectRepository {
         initialize();
     }
 
+    DefaultObjectRepository(EntityDecorator entityDecorator,
+                            NitriteCollection collection,
+                            NitriteConfig nitriteConfig) {
+        this.entityDecorator = entityDecorator;
+        this.collection = collection;
+        this.nitriteConfig = nitriteConfig;
+        initialize();
+    }
+
     @Override
     public void addProcessor(Processor processor) {
         notNull(processor, "a null processor cannot be added");
         collection.addProcessor(processor);
     }
 
-    @Override
-    public void removeProcessor(Processor processor) {
-        notNull(processor, "a null processor cannot be removed");
-        collection.removeProcessor(processor);
-    }
-
     @Override
     public void createIndex(IndexOptions indexOptions, String... fields) {
         collection.createIndex(indexOptions, fields);
@@ -111,17 +116,19 @@ public WriteResult insert(T[] elements) {
     @Override
     public WriteResult update(T element, boolean insertIfAbsent) {
         notNull(element, "a null object cannot be used for update");
-        return update(operations.createUniqueFilter(element), element, insertIfAbsent);
+        return update(operations.createUniqueFilter(element), element, updateOptions(insertIfAbsent, true));
     }
 
     @Override
-    public WriteResult update(Filter filter, T update, boolean insertIfAbsent) {
+    public WriteResult update(Filter filter, T update, UpdateOptions updateOptions) {
         notNull(update, "a null object cannot be used for update");
         Document updateDocument = operations.toDocument(update, true);
-        if (!insertIfAbsent) {
+        if (updateOptions == null || !updateOptions.isInsertIfAbsent()) {
             operations.removeNitriteId(updateDocument);
         }
-        return collection.update(operations.asObjectFilter(filter), updateDocument, updateOptions(insertIfAbsent, true));
+
+        return collection.update(operations.asObjectFilter(filter), updateDocument,
+            updateOptions);
     }
 
     @Override
@@ -151,7 +158,7 @@ public void clear() {
 
     @Override
     public Cursor find(Filter filter, FindOptions findOptions) {
-        return operations.find(filter, findOptions, type);
+        return operations.find(filter, findOptions, getType());
     }
 
     @Override
@@ -212,7 +219,11 @@ public void setAttributes(Attributes attributes) {
 
     @Override
     public Class getType() {
-        return type;
+        if (entityDecorator != null) {
+            return entityDecorator.getEntityType();
+        } else {
+            return type;
+        }
     }
 
     @Override
@@ -222,8 +233,11 @@ public NitriteCollection getDocumentCollection() {
 
     private void initialize() {
         NitriteMapper nitriteMapper = nitriteConfig.nitriteMapper();
-        operations = new RepositoryOperations(type, nitriteMapper, collection);
+        if (entityDecorator != null) {
+            operations = new RepositoryOperations(entityDecorator, nitriteMapper, collection);
+        } else {
+            operations = new RepositoryOperations(type, nitriteMapper, collection);
+        }
         operations.createIndices();
     }
-
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/EntityDecorator.java b/nitrite/src/main/java/org/dizitart/no2/repository/EntityDecorator.java
new file mode 100644
index 000000000..fee8de619
--- /dev/null
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/EntityDecorator.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.repository;
+
+import java.util.List;
+
+/**
+ * A class that implements this interface can be used to decorate
+ * an entity of type T for nitrite database where using
+ * {@link org.dizitart.no2.repository.annotations.Entity}
+ * or its related annotations is not possible on a class.
+ *
+ * @param  the type parameter
+ * @see org.dizitart.no2.Nitrite#getRepository(EntityDecorator)
+ * @see org.dizitart.no2.Nitrite#getRepository(EntityDecorator, String)
+ * @since 4.0
+ * @author Anindya Chatterjee
+ */
+public interface EntityDecorator {
+    /**
+     * Gets the entity type of the decorator.
+     *
+     * @return the entity type
+     */
+    Class getEntityType();
+
+    /**
+     * Gets id field declaration.
+     *
+     * @return the id field
+     */
+    EntityId getIdField();
+
+    /**
+     * Gets index fields declaration.
+     *
+     * @return the index fields
+     */
+    List getIndexFields();
+
+    /**
+     * Gets entity name.
+     *
+     * @return the entity name
+     */
+    default String getEntityName() {
+        return getEntityType().getName();
+    }
+}
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/EntityDecoratorScanner.java b/nitrite/src/main/java/org/dizitart/no2/repository/EntityDecoratorScanner.java
new file mode 100644
index 000000000..46667e643
--- /dev/null
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/EntityDecoratorScanner.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.repository;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.util.StringUtils;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.dizitart.no2.index.IndexOptions.indexOptions;
+
+/**
+ * @author Anindya Chatterjee
+ */
+public class EntityDecoratorScanner {
+    private final EntityDecorator entityDecorator;
+    private final NitriteCollection collection;
+    private final NitriteMapper nitriteMapper;
+    private final IndexValidator indexValidator;
+
+    private final Reflector reflector;
+
+    @Getter(AccessLevel.PACKAGE)
+    private final Set indices;
+
+    @Getter
+    private ObjectIdField objectIdField;
+
+    public EntityDecoratorScanner(EntityDecorator entityDecorator,
+                                  NitriteCollection collection,
+                                  NitriteMapper nitriteMapper) {
+        this.entityDecorator = entityDecorator;
+        this.collection = collection;
+        this.nitriteMapper = nitriteMapper;
+        this.indices = new HashSet<>();
+        this.indexValidator = new IndexValidator();
+        this.reflector = new Reflector();
+    }
+
+    public void readEntity() {
+        readIndices();
+        readIdField();
+    }
+
+    public void createIndices() {
+        for (EntityIndex index : indices) {
+            String[] fields = index.getFieldNames().toArray(new String[0]);
+            if (!collection.hasIndex(fields)) {
+                collection.createIndex(indexOptions(index.getIndexType()), fields);
+            }
+        }
+    }
+
+    public void createIdIndex() {
+        if (objectIdField != null) {
+            String[] fieldNames = objectIdField.getEmbeddedFieldNames();
+            if (!collection.hasIndex(fieldNames)) {
+                collection.createIndex(fieldNames);
+            }
+        }
+    }
+
+    private void readIndices() {
+        if (entityDecorator.getIndexFields() != null) {
+            for (EntityIndex indexField : entityDecorator.getIndexFields()) {
+                List names = indexField.getFieldNames();
+                List entityFields = new ArrayList<>();
+
+                for (String name : names) {
+                    Field field = reflector.getField(entityDecorator.getEntityType(), name);
+                    if (field != null) {
+                        entityFields.add(field);
+                        indexValidator.validate(field.getType(), field.getName(), nitriteMapper);
+                    }
+                }
+
+                if (entityFields.size() == names.size()) {
+                    // validation for all field are success
+                    indices.add(indexField);
+                }
+            }
+        }
+    }
+
+    private void readIdField() {
+        if (entityDecorator != null) {
+            EntityId entityId = entityDecorator.getIdField();
+            if (entityId != null) {
+                String idFieldName = entityId.getFieldName();
+                if (!StringUtils.isNullOrEmpty(idFieldName)) {
+                    Field field = reflector.getField(entityDecorator.getEntityType(), idFieldName);
+                    indexValidator.validateId(entityId, field.getType(), idFieldName, nitriteMapper);
+
+                    objectIdField = new ObjectIdField();
+                    objectIdField.setField(field);
+                    objectIdField.setIdFieldName(idFieldName);
+                    objectIdField.setEmbedded(entityId.isEmbedded());
+                    objectIdField.setFieldNames(entityId.getSubFields());
+                }
+            }
+        }
+    }
+}
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/EntityId.java b/nitrite/src/main/java/org/dizitart/no2/repository/EntityId.java
new file mode 100644
index 000000000..99921a6f9
--- /dev/null
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/EntityId.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.repository;
+
+import lombok.Getter;
+import org.dizitart.no2.NitriteConfig;
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.exceptions.ObjectMappingException;
+import org.dizitart.no2.filters.Filter;
+import org.dizitart.no2.filters.NitriteFilter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.dizitart.no2.filters.FluentFilter.where;
+
+public class EntityId {
+    @Getter
+    private String fieldName;
+
+    @Getter
+    private String[] subFields;
+
+    private List embeddedFieldNames;
+
+    public EntityId(String fieldName, String... subFields) {
+        this.fieldName = fieldName;
+        this.subFields = subFields;
+    }
+
+    public List getEmbeddedFieldNames() {
+        if (embeddedFieldNames != null) return embeddedFieldNames;
+        embeddedFieldNames = new ArrayList<>();
+
+        if (subFields != null) {
+            for (String subField : subFields) {
+                embeddedFieldNames.add(fieldName + NitriteConfig.getFieldSeparator() + subField);
+            }
+        }
+        return embeddedFieldNames;
+    }
+
+    public boolean isEmbedded() {
+        return subFields != null && subFields.length != 0;
+    }
+
+    public Filter createUniqueFilter(Object value, NitriteMapper nitriteMapper) {
+        if (isEmbedded()) {
+            Document document = nitriteMapper.convert(value, Document.class);
+            if (document == null) {
+                throw new ObjectMappingException("Failed to map object to document");
+            }
+
+            List filters = new ArrayList<>();
+            for (String subField : subFields) {
+                String filterField = fieldName + NitriteConfig.getFieldSeparator() + subField;
+                Object fieldValue = document.get(subField);
+                filters.add(where(filterField).eq(fieldValue));
+            }
+
+            NitriteFilter nitriteFilter = (NitriteFilter) Filter.and(filters.toArray(new Filter[] {}));
+            nitriteFilter.setObjectFilter(true);
+            return nitriteFilter;
+        } else {
+            return where(fieldName).eq(value);
+        }
+    }
+}
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/EntityIndex.java b/nitrite/src/main/java/org/dizitart/no2/repository/EntityIndex.java
new file mode 100644
index 000000000..1342bfd36
--- /dev/null
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/EntityIndex.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.repository;
+
+import lombok.Getter;
+import org.dizitart.no2.common.util.StringUtils;
+import org.dizitart.no2.index.IndexType;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.dizitart.no2.common.util.ValidationUtils.notEmpty;
+import static org.dizitart.no2.common.util.ValidationUtils.notNull;
+
+/**
+ * Represents an index fields.
+ *
+ * @author Anindya Chatterjee
+ * @since 4.0
+ */
+public class EntityIndex {
+
+    @Getter
+    private String indexType;
+
+    @Getter
+    private List fieldNames;
+
+    public EntityIndex(String indexType, String... fields) {
+        notNull(fields, "fields cannot be null");
+        notEmpty(fields, "fields cannot be empty");
+
+        if (StringUtils.isNullOrEmpty(indexType)) {
+            indexType = IndexType.UNIQUE;
+        }
+
+        this.fieldNames = Arrays.asList(fields);
+        this.indexType = StringUtils.isNullOrEmpty(indexType) ? IndexType.UNIQUE : indexType;
+    }
+}
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/IndexValidator.java b/nitrite/src/main/java/org/dizitart/no2/repository/IndexValidator.java
index f18b42e03..470788ba0 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/IndexValidator.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/IndexValidator.java
@@ -20,24 +20,16 @@
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.util.ObjectUtils;
 import org.dizitart.no2.exceptions.IndexingException;
-import org.dizitart.no2.repository.annotations.Embedded;
+import org.dizitart.no2.repository.annotations.Id;
 
-import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
-import java.util.List;
-
-import static org.dizitart.no2.common.util.DocumentUtils.skeletonDocument;
 
 /**
  * @author Anindya Chatterjee
  */
 public class IndexValidator {
-    private final Reflector reflector;
-
-    public IndexValidator(Reflector reflector) {
-        this.reflector = reflector;
-    }
 
     /**
      * Validate an index field of an {@link org.dizitart.no2.repository.annotations.Entity} object.
@@ -50,7 +42,7 @@ public void validate(Class fieldType, String field, NitriteMapper nitriteMapp
         if (fieldType.isPrimitive()
             || fieldType == NitriteId.class
             || fieldType.isInterface()
-            || nitriteMapper.isValueType(fieldType)
+//            || ObjectUtils.isValueType(fieldType, nitriteMapper)
             || Modifier.isAbstract(fieldType.getModifiers())
             || fieldType.isArray()
             || Iterable.class.isAssignableFrom(fieldType)) {
@@ -58,32 +50,36 @@ public void validate(Class fieldType, String field, NitriteMapper nitriteMapp
             return;
         }
 
-        Document document;
-        try {
-            document = skeletonDocument(nitriteMapper, fieldType);
-            if (document.size() > 0) {
-                // compound index
-                boolean embeddedFieldFound = false;
-                List fields = reflector.getAllFields(fieldType);
-                for (Field indexField : fields) {
-                    if (indexField.isAnnotationPresent(Embedded.class)) {
-                        embeddedFieldFound = true;
-                        break;
-                    }
-                }
+        if (!Comparable.class.isAssignableFrom(fieldType)) {
+            throw new IndexingException("Cannot create index on non comparable field " + field);
+        }
+    }
+
+    public void validateId(Id id, Class fieldType, String field, NitriteMapper nitriteMapper) {
+        if (fieldType.isPrimitive()
+            || fieldType == NitriteId.class) {
+            return;
+        }
+
+        Object dummyValue = ObjectUtils.newInstance(fieldType, true, nitriteMapper);
+        Document dummyDocument = nitriteMapper.convert(dummyValue, Document.class);
+
+        if (dummyDocument != null && dummyDocument.size() != 0 && id.embeddedFields().length == 0) {
+            throw new IndexingException("Invalid Id field " + field);
+        }
+    }
+
+    public void validateId(EntityId entityId, Class fieldType, String field, NitriteMapper nitriteMapper) {
+        if (fieldType.isPrimitive()
+            || fieldType == NitriteId.class) {
+            return;
+        }
+
+        Object dummyValue = ObjectUtils.newInstance(fieldType, true, nitriteMapper);
+        Document dummyDocument = nitriteMapper.convert(dummyValue, Document.class);
 
-                if (!embeddedFieldFound) {
-                    throw new IndexingException("no embedded field found for object id");
-                }
-            } else {
-                if (!Comparable.class.isAssignableFrom(fieldType)) {
-                    throw new IndexingException("cannot index on non comparable field " + field);
-                }
-            }
-        } catch (IndexingException ie) {
-            throw ie;
-        } catch (Throwable e) {
-            throw new IndexingException("invalid type specified " + fieldType.getName() + " for indexing", e);
+        if (dummyDocument.size() != 0 && entityId.getEmbeddedFieldNames().size() == 0) {
+            throw new IndexingException("Invalid Id field " + field);
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/ObjectCursor.java b/nitrite/src/main/java/org/dizitart/no2/repository/ObjectCursor.java
index 0087a6b6c..e923a1db4 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/ObjectCursor.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/ObjectCursor.java
@@ -21,15 +21,17 @@
 import org.dizitart.no2.collection.FindPlan;
 import org.dizitart.no2.common.Lookup;
 import org.dizitart.no2.common.RecordStream;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.streams.MutatedObjectStream;
 import org.dizitart.no2.exceptions.InvalidOperationException;
 import org.dizitart.no2.exceptions.ValidationException;
-import org.dizitart.no2.common.mapper.NitriteMapper;
 
 import java.lang.reflect.Modifier;
 import java.util.Iterator;
 
 import static org.dizitart.no2.common.util.DocumentUtils.skeletonDocument;
 import static org.dizitart.no2.common.util.ValidationUtils.notNull;
+import static org.dizitart.no2.common.util.ValidationUtils.validateProjectionType;
 
 /**
  * @author Anindya Chatterjee.
@@ -75,20 +77,20 @@ public Iterator iterator() {
 
     private  Document emptyDocument(NitriteMapper nitriteMapper, Class type) {
         if (type.isPrimitive()) {
-            throw new ValidationException("cannot project to primitive type");
+            throw new ValidationException("Cannot project to primitive type");
         } else if (type.isInterface()) {
-            throw new ValidationException("cannot project to interface");
+            throw new ValidationException("Cannot project to interface");
         } else if (type.isArray()) {
-            throw new ValidationException("cannot project to array");
+            throw new ValidationException("Cannot project to array");
         } else if (Modifier.isAbstract(type.getModifiers())) {
-            throw new ValidationException("cannot project to abstract type");
-        } else if (nitriteMapper.isValueType(type)) {
-            throw new ValidationException("cannot to project to nitrite mapper's value type");
+            throw new ValidationException("Cannot project to abstract type");
         }
 
+        validateProjectionType(type, nitriteMapper);
+
         Document dummyDoc = skeletonDocument(nitriteMapper, type);
-        if (dummyDoc == null) {
-            throw new ValidationException("cannot project to empty type");
+        if (dummyDoc == null || dummyDoc.size() == 0) {
+            throw new ValidationException("Cannot project to empty type");
         } else {
             return dummyDoc;
         }
@@ -114,7 +116,7 @@ public T next() {
 
         @Override
         public void remove() {
-            throw new InvalidOperationException("remove on a cursor is not supported");
+            throw new InvalidOperationException("Remove on a cursor is not supported");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/ObjectIdField.java b/nitrite/src/main/java/org/dizitart/no2/repository/ObjectIdField.java
index 1061273cd..161b36751 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/ObjectIdField.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/ObjectIdField.java
@@ -22,16 +22,10 @@
 import org.dizitart.no2.NitriteConfig;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.common.mapper.NitriteMapper;
-import org.dizitart.no2.common.util.StringUtils;
-import org.dizitart.no2.exceptions.IndexingException;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.filters.NitriteFilter;
-import org.dizitart.no2.repository.annotations.Embedded;
 
 import java.lang.reflect.Field;
-import java.util.List;
-import java.util.NavigableMap;
-import java.util.TreeMap;
 
 import static org.dizitart.no2.filters.FluentFilter.where;
 
@@ -39,9 +33,9 @@
  * @author Anindya Chatterjee
  */
 class ObjectIdField {
-    private final Reflector reflector;
-    private final IndexValidator indexValidator;
-    private String[] embeddedFieldNames;
+    @Getter
+    @Setter
+    private String[] fieldNames;
 
     @Getter
     @Setter
@@ -55,60 +49,32 @@ class ObjectIdField {
     @Setter
     private String idFieldName;
 
-    public ObjectIdField() {
-        this.reflector = new Reflector();
-        this.indexValidator = new IndexValidator(reflector);
-    }
-
-    public String[] getFieldNames(NitriteMapper nitriteMapper) {
-        if (embeddedFieldNames != null) {
-            return embeddedFieldNames;
-        }
-
+    public String[] getEmbeddedFieldNames() {
         if (!isEmbedded) {
-            embeddedFieldNames = new String[]{ idFieldName };
-            return embeddedFieldNames;
-        }
-
-        List fieldList = reflector.getAllFields(field.getType());
-        NavigableMap orderedFieldName = new TreeMap<>();
-
-        boolean embeddedFieldFound = false;
-        for (Field field : fieldList) {
-            if (field.isAnnotationPresent(Embedded.class)) {
-                embeddedFieldFound = true;
-                Embedded embedded = field.getAnnotation(Embedded.class);
-                int order = embedded.order();
-                String fieldName = StringUtils.isNullOrEmpty(embedded.fieldName())
-                    ? field.getName() : embedded.fieldName();
-
-                String name = this.idFieldName + NitriteConfig.getFieldSeparator() + fieldName;
-                indexValidator.validate(field.getType(), name, nitriteMapper);
-
-                orderedFieldName.put(order, name);
-            }
+            return new String[]{ idFieldName };
         }
 
-        if (!embeddedFieldFound) {
-            throw new IndexingException("no embedded field found for " + field.getName());
+        String[] fieldNames = new String[this.fieldNames.length];
+        for (int i = 0; i < this.fieldNames.length; i++) {
+            String name = this.idFieldName + NitriteConfig.getFieldSeparator() + this.fieldNames[i];
+            fieldNames[i] = name;
         }
 
-        embeddedFieldNames = orderedFieldName.values().toArray(new String[0]);
-        return embeddedFieldNames;
+        return fieldNames;
     }
 
     public Filter createUniqueFilter(Object value, NitriteMapper nitriteMapper) {
-        if (embeddedFieldNames.length == 1) {
+        if (getEmbeddedFieldNames().length == 1) {
             return where(idFieldName).eq(value);
         } else {
             Document document = nitriteMapper.convert(value, Document.class);
-            Filter[] filters = new Filter[embeddedFieldNames.length];
+            Filter[] filters = new Filter[fieldNames.length];
 
             int index = 0;
-            for (String field : embeddedFieldNames) {
-                String docFieldName = getEmbeddedFieldName(field);
-                Object fieldValue = document.get(docFieldName);
-                filters[index++] = where(field).eq(fieldValue);
+            for (String field : fieldNames) {
+                String filterField = idFieldName + NitriteConfig.getFieldSeparator() + field;
+                Object fieldValue = document.get(field);
+                filters[index++] = where(filterField).eq(fieldValue);
             }
 
             NitriteFilter nitriteFilter = (NitriteFilter) Filter.and(filters);
@@ -116,12 +82,4 @@ public Filter createUniqueFilter(Object value, NitriteMapper nitriteMapper) {
             return nitriteFilter;
         }
     }
-
-    private String getEmbeddedFieldName(String fieldName) {
-        if (fieldName.contains(NitriteConfig.getFieldSeparator())) {
-            return fieldName.substring(fieldName.indexOf(NitriteConfig.getFieldSeparator()) + 1);
-        } else {
-            return fieldName;
-        }
-    }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/ObjectRepository.java b/nitrite/src/main/java/org/dizitart/no2/repository/ObjectRepository.java
index 48c01be4f..b22ed32e7 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/ObjectRepository.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/ObjectRepository.java
@@ -16,10 +16,7 @@
 
 package org.dizitart.no2.repository;
 
-import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.collection.FindOptions;
-import org.dizitart.no2.collection.NitriteCollection;
-import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.collection.*;
 import org.dizitart.no2.collection.events.CollectionEventListener;
 import org.dizitart.no2.collection.events.EventAware;
 import org.dizitart.no2.collection.events.EventType;
@@ -37,6 +34,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.dizitart.no2.collection.UpdateOptions.updateOptions;
 import static org.dizitart.no2.common.util.ValidationUtils.containsNull;
 import static org.dizitart.no2.common.util.ValidationUtils.notNull;
 
@@ -78,9 +76,7 @@ public interface ObjectRepository extends PersistentCollection {
     /**
      * Inserts object into this repository. If the object contains a value marked with
      * {@link Id}, then the value will be used as a unique key to identify the object
-     * in the repository. If the object does not contain any value marked with {@link Id},
-     * then nitrite will generate a new {@link NitriteId} and will add it to the document
-     * generated from the object.
+     * in the repository.
      * 

      * If any of the value is already indexed in the repository, then after insertion the
      * index will also be updated.
@@ -112,7 +108,7 @@ default WriteResult insert(T object, T... others) {
         List itemList = new ArrayList<>();
         itemList.add(object);
 
-        if (others != null && itemList.size() > 0) {
+        if (others != null) {
             Collections.addAll(itemList, others);
         }
 
@@ -141,7 +137,7 @@ default WriteResult insert(T object, T... others) {
      * @throws ValidationException if the {@code update} object is {@code null}.
      */
     default WriteResult update(Filter filter, T update) {
-        return update(filter, update, false);
+        return update(filter, update, updateOptions(false));
     }
 
     /**
@@ -163,12 +159,12 @@ default WriteResult update(Filter filter, T update) {
      *
      * @param filter         the filter to apply to select objects from the collection.
      * @param update         the modifications to apply.
-     * @param insertIfAbsent if set to {@code true}, {@code update} object will be inserted if not found.
+     * @param updateOptions  various options for update operation.
      * @return the result of the update operation.
      * @throws ValidationException if the {@code update} object is {@code null}.
      * @throws ValidationException if {@code updateOptions} is {@code null}.
      */
-    WriteResult update(Filter filter, T update, boolean insertIfAbsent);
+    WriteResult update(Filter filter, T update, UpdateOptions updateOptions);
 
     /**
      * Updates object in the repository by setting the field specified in {@code document}.
@@ -328,7 +324,7 @@ default Cursor find(FindOptions findOptions) {
     Class getType();
 
     /**
-     * Returns the underlying document collection.
+     * Returns the underlying {@link NitriteCollection} instance.
      *
      * @return the underlying document collection.
      */
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/Reflector.java b/nitrite/src/main/java/org/dizitart/no2/repository/Reflector.java
index b229cfeca..fa334668c 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/Reflector.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/Reflector.java
@@ -67,14 +67,14 @@ public  Field getEmbeddedField(Class startingClass, String embeddedField)
         String remaining = split.length == 2 ? split[1] : "";
 
         if (isNullOrEmpty(key)) {
-            throw new ValidationException("invalid embedded field provided");
+            throw new ValidationException("Invalid embedded field provided");
         }
 
         Field field;
         try {
             field = startingClass.getDeclaredField(key);
         } catch (NoSuchFieldException e) {
-            throw new ValidationException("no such field '" + key + "' for type " + startingClass.getName(), e);
+            throw new ValidationException("No such field '" + key + "' for type " + startingClass.getName(), e);
         }
 
         if (!isNullOrEmpty(remaining) || remaining.contains(NitriteConfig.getFieldSeparator())) {
@@ -123,7 +123,7 @@ public  Field getField(Class type, String name) {
                 }
             }
             if (field == null) {
-                throw new ValidationException("no such field '" + name + "' for type " + type.getName());
+                throw new ValidationException("No such field '" + name + "' for type " + type.getName());
             }
             return field;
         }
@@ -139,7 +139,7 @@ public  List getAllFields(Class type) {
         return fields;
     }
 
-    public void filterSynthetics(List fields) {
+    private void filterSynthetics(List fields) {
         if (fields == null || fields.isEmpty()) return;
         Iterator iterator = fields.iterator();
         if (iterator.hasNext()) {
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryFactory.java b/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryFactory.java
index 2fe8de751..f3bf6f642 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryFactory.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryFactory.java
@@ -19,10 +19,10 @@
 import org.dizitart.no2.NitriteConfig;
 import org.dizitart.no2.collection.CollectionFactory;
 import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.common.util.StringUtils;
 import org.dizitart.no2.exceptions.NitriteIOException;
 import org.dizitart.no2.exceptions.ValidationException;
-import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.store.NitriteStore;
 import org.dizitart.no2.store.StoreCatalog;
 
@@ -31,6 +31,8 @@
 import java.util.concurrent.locks.ReentrantLock;
 
 import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryName;
+import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryNameByDecorator;
+import static org.dizitart.no2.common.util.ValidationUtils.validateRepositoryType;
 
 /**
  * The {@link ObjectRepository} factory.
@@ -105,6 +107,41 @@ public  ObjectRepository getRepository(NitriteConfig nitriteConfig, Class<
         }
     }
 
+
+    public  ObjectRepository getRepository(NitriteConfig nitriteConfig, EntityDecorator entityDecorator) {
+        return getRepository(nitriteConfig, entityDecorator, null);
+    }
+
+    @SuppressWarnings("unchecked")
+    public  ObjectRepository getRepository(NitriteConfig nitriteConfig, EntityDecorator entityDecorator, String key) {
+        if (entityDecorator == null) {
+            throw new ValidationException("entityDecorator cannot be null");
+        }
+
+        if (nitriteConfig == null) {
+            throw new ValidationException("nitriteConfig cannot be null");
+        }
+
+        String collectionName = findRepositoryNameByDecorator(entityDecorator, key);
+
+        try {
+            lock.lock();
+            if (repositoryMap.containsKey(collectionName)) {
+                ObjectRepository repository = (ObjectRepository) repositoryMap.get(collectionName);
+                if (repository.isDropped() || !repository.isOpen()) {
+                    repositoryMap.remove(collectionName);
+                    return createRepositoryByDecorator(nitriteConfig, entityDecorator, collectionName, key);
+                } else {
+                    return repository;
+                }
+            } else {
+                return createRepositoryByDecorator(nitriteConfig, entityDecorator, collectionName, key);
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
     /**
      * Closes all opened {@link ObjectRepository}s and clear internal data from this class.
      */
@@ -116,7 +153,7 @@ public void clear() {
             }
             repositoryMap.clear();
         } catch (Exception e) {
-            throw new NitriteIOException("failed to close an object repository", e);
+            throw new NitriteIOException("Failed to clear an object repository", e);
         } finally {
             lock.unlock();
         }
@@ -126,12 +163,11 @@ private  ObjectRepository createRepository(NitriteConfig nitriteConfig, Cl
                                                      String collectionName, String key) {
         NitriteMapper nitriteMapper = nitriteConfig.nitriteMapper();
         NitriteStore store = nitriteConfig.getNitriteStore();
-        if (nitriteMapper.isValueType(type)) {
-            throw new ValidationException("a value type cannot be used to create repository");
-        }
+
+        validateRepositoryType(type, nitriteMapper);
 
         if (store.getCollectionNames().contains(collectionName)) {
-            throw new ValidationException("a collection with same entity name already exists");
+            throw new ValidationException("A collection with same entity name already exists");
         }
 
         NitriteCollection nitriteCollection = collectionFactory.getCollection(collectionName,
@@ -143,12 +179,34 @@ private  ObjectRepository createRepository(NitriteConfig nitriteConfig, Cl
         return repository;
     }
 
+    private  ObjectRepository createRepositoryByDecorator(NitriteConfig nitriteConfig,
+                                                                EntityDecorator entityDecorator,
+                                                                String collectionName, String key) {
+        NitriteMapper nitriteMapper = nitriteConfig.nitriteMapper();
+        NitriteStore store = nitriteConfig.getNitriteStore();
+
+        if (store.getCollectionNames().contains(collectionName)) {
+            throw new ValidationException("A collection with same entity name already exists");
+        }
+
+        validateRepositoryType(entityDecorator.getEntityType(), nitriteMapper);
+
+        NitriteCollection nitriteCollection = collectionFactory.getCollection(collectionName,
+            nitriteConfig, false);
+
+        ObjectRepository repository = new DefaultObjectRepository<>(entityDecorator, nitriteCollection, nitriteConfig);
+        repositoryMap.put(collectionName, repository);
+
+        writeCatalog(store, collectionName, key);
+        return repository;
+    }
+
     private void writeCatalog(NitriteStore store, String name, String key) {
         StoreCatalog storeCatalog = store.getCatalog();
         if (StringUtils.isNullOrEmpty(key)) {
             storeCatalog.writeRepositoryEntry(name);
         } else {
-            storeCatalog.writeKeyedRepositoryEntries(name);
+            storeCatalog.writeKeyedRepositoryEntry(name);
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryOperations.java b/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryOperations.java
index eb4e2703b..f25a25d9e 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryOperations.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/RepositoryOperations.java
@@ -21,6 +21,7 @@
 import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.exceptions.InvalidIdException;
 import org.dizitart.no2.exceptions.NotIdentifiableException;
+import org.dizitart.no2.exceptions.ObjectMappingException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.filters.NitriteFilter;
@@ -42,10 +43,11 @@
  */
 public class RepositoryOperations {
     private final NitriteMapper nitriteMapper;
-    private final Class type;
     private final NitriteCollection collection;
-    private final AnnotationScanner annotationScanner;
+    private final Class type;
+    private AnnotationScanner annotationScanner;
     private ObjectIdField objectIdField;
+    private EntityDecoratorScanner entityDecoratorScanner;
 
     /**
      * Instantiates a new {@link RepositoryOperations}.
@@ -55,8 +57,8 @@ public class RepositoryOperations {
      * @param collection    the collection
      */
     public RepositoryOperations(Class type,
-                                NitriteMapper nitriteMapper,
-                                NitriteCollection collection) {
+            NitriteMapper nitriteMapper,
+            NitriteCollection collection) {
         this.type = type;
         this.nitriteMapper = nitriteMapper;
         this.collection = collection;
@@ -64,14 +66,31 @@ public RepositoryOperations(Class type,
         validateCollection();
     }
 
+    public RepositoryOperations(EntityDecorator entityDecorator,
+                                NitriteMapper nitriteMapper,
+                                NitriteCollection collection) {
+        this.type = entityDecorator.getEntityType();
+        this.nitriteMapper = nitriteMapper;
+        this.collection = collection;
+        this.entityDecoratorScanner = new EntityDecoratorScanner(entityDecorator, collection, nitriteMapper);
+        validateCollection();
+    }
+
     /**
      * Create indices.
      */
     public void createIndices() {
-        annotationScanner.scanIndices();
-        annotationScanner.createIndices();
-        annotationScanner.createIdIndex();
-        objectIdField = annotationScanner.getObjectIdField();
+        if (annotationScanner != null) {
+            annotationScanner.performScan();
+            annotationScanner.createIndices();
+            annotationScanner.createIdIndex();
+            objectIdField = annotationScanner.getObjectIdField();
+        } else if (entityDecoratorScanner != null) {
+            entityDecoratorScanner.readEntity();
+            entityDecoratorScanner.createIndices();
+            entityDecoratorScanner.createIdIndex();
+            objectIdField = entityDecoratorScanner.getObjectIdField();
+        }
     }
 
     /**
@@ -99,10 +118,11 @@ public void serializeFields(Document document) {
      * @return the document [ ]
      */
     public  Document[] toDocuments(T[] others) {
-        if (others == null || others.length == 0) return null;
+        if (others == null || others.length == 0)
+            return null;
         Document[] documents = new Document[others.length];
         for (int i = 0; i < others.length; i++) {
-            documents[i] = toDocument(others[i], false);
+            documents[i] = toDocument(others[i], false); // this method is for insert only
         }
         return documents;
     }
@@ -117,6 +137,9 @@ public  Document[] toDocuments(T[] others) {
      */
     public  Document toDocument(T object, boolean update) {
         Document document = nitriteMapper.convert(object, Document.class);
+        if (document == null) {
+            throw new ObjectMappingException("Failed to map object to document");
+        }
 
         if (objectIdField != null) {
             Field idField = objectIdField.getField();
@@ -129,19 +152,21 @@ public  Document toDocument(T object, boolean update) {
                         idField.set(object, id);
                         document.put(objectIdField.getIdFieldName(), nitriteMapper.convert(id, Comparable.class));
                     } else if (!update) {
-                        throw new InvalidIdException("auto generated id should not be set manually");
+                        // if it is an insert, then we should not allow to insert the document with user
+                        // provided id
+                        throw new InvalidIdException("Auto generated id should not be set manually");
                     }
                 } catch (IllegalAccessException iae) {
-                    throw new InvalidIdException("auto generated id value cannot be accessed");
+                    throw new InvalidIdException("Auto generated id value cannot be accessed");
                 }
             }
 
             Object idValue = document.get(objectIdField.getIdFieldName());
             if (idValue == null) {
-                throw new InvalidIdException("id cannot be null");
+                throw new InvalidIdException("Id cannot be null");
             }
             if (idValue instanceof String && isNullOrEmpty((String) idValue)) {
-                throw new InvalidIdException("id value cannot be empty string");
+                throw new InvalidIdException("Id value cannot be empty string");
             }
         }
         return document;
@@ -155,7 +180,7 @@ public  Document toDocument(T object, boolean update) {
      */
     public Filter createUniqueFilter(Object object) {
         if (objectIdField == null) {
-            throw new NotIdentifiableException("update operation failed as no id value found for the object");
+            throw new NotIdentifiableException("No id value found for the object");
         }
 
         Field idField = objectIdField.getField();
@@ -163,11 +188,11 @@ public Filter createUniqueFilter(Object object) {
         try {
             Object value = idField.get(object);
             if (value == null) {
-                throw new InvalidIdException("id value cannot be null");
+                throw new InvalidIdException("Id value cannot be null");
             }
             return objectIdField.createUniqueFilter(value, nitriteMapper);
         } catch (IllegalAccessException iae) {
-            throw new InvalidIdException("id field is not accessible");
+            throw new InvalidIdException("Id field is not accessible");
         }
     }
 
@@ -181,7 +206,7 @@ public void removeNitriteId(Document document) {
         if (objectIdField != null) {
             Field idField = objectIdField.getField();
             if (idField != null && !objectIdField.isEmbedded()
-                && idField.getType() == NitriteId.class) {
+                    && idField.getType() == NitriteId.class) {
                 document.remove(idField.getName());
             }
         }
@@ -197,10 +222,10 @@ public void removeNitriteId(Document document) {
     public  Filter createIdFilter(I id) {
         if (objectIdField != null) {
             if (id == null) {
-                throw new InvalidIdException("a null id is not a valid id");
+                throw new InvalidIdException("Id cannot be null");
             }
             if (!isCompatibleTypes(id.getClass(), objectIdField.getField().getType())) {
-                throw new InvalidIdException("a value of invalid type is provided as id");
+                throw new InvalidIdException("A value of invalid type is provided as id");
             }
 
             return objectIdField.createUniqueFilter(id, nitriteMapper);
@@ -240,7 +265,7 @@ public  Cursor find(Filter filter, FindOptions findOptions, Class type)
 
     private void validateCollection() {
         if (collection == null) {
-            throw new ValidationException("repository has not been initialized properly");
+            throw new ValidationException("Repository has not been initialized properly");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Embedded.java b/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Embedded.java
deleted file mode 100644
index 70f680d40..000000000
--- a/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Embedded.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2017-2021 Nitrite author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.dizitart.no2.repository.annotations;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * Indicates that an annotated field is used to construct a composite id field.
- *
- * @author Anindya Chatterjee
- * @since 4.0.0
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.FIELD)
-public @interface Embedded {
-    /**
-     * Order of the field in compound index.
-     *
-     * @return the int
-     */
-    int order();
-
-    /**
-     * The custom field name in compound index.
-     *
-     * @return the string
-     */
-    String fieldName() default "";
-}
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Id.java b/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Id.java
index b8d8f6a1b..afd9c5178 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Id.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Id.java
@@ -36,4 +36,11 @@
      * @return the string
      */
     String fieldName() default "";
+
+    /**
+     * The name of the embedded fields.
+     *
+     * @return the string
+     */
+    String[] embeddedFields() default {};
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Index.java b/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Index.java
index 635e5fd33..5364fcda5 100644
--- a/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Index.java
+++ b/nitrite/src/main/java/org/dizitart/no2/repository/annotations/Index.java
@@ -35,7 +35,7 @@
      *
      * @return the field name
      */
-    String[] value();
+    String[] fields();
 
     /**
      * Type of the index.
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/AbstractNitriteStore.java b/nitrite/src/main/java/org/dizitart/no2/store/AbstractNitriteStore.java
index 87a4f084b..8f0c92fb5 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/AbstractNitriteStore.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/AbstractNitriteStore.java
@@ -47,7 +47,7 @@ protected AbstractNitriteStore() {
     }
 
     /**
-     * Alerts about an {@link StoreEvents} to all subscribed {@link StoreEventListener}s.
+     * Alerts about a {@link StoreEvents} to all subscribed {@link StoreEventListener}s.
      *
      * @param eventType the event type
      */
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/NitriteMap.java b/nitrite/src/main/java/org/dizitart/no2/store/NitriteMap.java
index 5711fd77f..f55a3127b 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/NitriteMap.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/NitriteMap.java
@@ -16,8 +16,8 @@
 
 package org.dizitart.no2.store;
 
-import org.dizitart.no2.collection.meta.Attributes;
-import org.dizitart.no2.collection.meta.MetadataAware;
+import org.dizitart.no2.common.meta.Attributes;
+import org.dizitart.no2.common.meta.AttributesAware;
 import org.dizitart.no2.common.RecordStream;
 import org.dizitart.no2.common.tuples.Pair;
 
@@ -33,7 +33,7 @@
  * @author Anindya Chatterjee.
  * @since 1.0
  */
-public interface NitriteMap extends MetadataAware, AutoCloseable {
+public interface NitriteMap extends AttributesAware, AutoCloseable {
     /**
      * Determines if the map contains a mapping for the
      * specified key.
@@ -56,6 +56,13 @@ public interface NitriteMap extends MetadataAware, AutoCloseable {
      */
     void clear();
 
+    /**
+     * Indicates if the map already is closed.
+     *
+     * @return the boolean
+     */
+    boolean isClosed();
+
     /**
      * Closes this {@link NitriteMap}.
      * */
@@ -95,8 +102,8 @@ public interface NitriteMap extends MetadataAware, AutoCloseable {
     void put(Key key, Value value);
 
     /**
-     * Get the number of entries, as a integer. Integer.MAX_VALUE is returned if
-     * there are more than this entries.
+     * Get the number of entries, as an integer. Integer.MAX_VALUE is returned if
+     * there are more than these entries.
      *
      * @return the number of entries, as an integer.
      */
@@ -112,7 +119,7 @@ public interface NitriteMap extends MetadataAware, AutoCloseable {
     Value putIfAbsent(Key key, Value value);
 
     /**
-     * Get the smallest key that is larger than the given key, or null if no
+     * Get the lest key that is greater than the given key, or null if no
      * such key exists.
      *
      * @param key the key
@@ -121,7 +128,7 @@ public interface NitriteMap extends MetadataAware, AutoCloseable {
     Key higherKey(Key key);
 
     /**
-     * Get the smallest key that is larger or equal to this key.
+     * Get the least key that is greater than or equal to this key.
      *
      * @param key the key
      * @return the result.
@@ -185,13 +192,22 @@ public interface NitriteMap extends MetadataAware, AutoCloseable {
      */
     void drop();
 
+    /**
+     * Indicates if this map is dropped already.
+     *
+     * @return the boolean result
+     */
+    boolean isDropped();
+
     /**
      * Gets the attributes of this map.
      * */
     default Attributes getAttributes() {
-        NitriteMap metaMap = getStore().openMap(META_MAP_NAME, String.class, Attributes.class);
-        if (metaMap != null && !getName().contentEquals(META_MAP_NAME)) {
-            return metaMap.get(getName());
+        if (!isDropped()) {
+            NitriteMap metaMap = getStore().openMap(META_MAP_NAME, String.class, Attributes.class);
+            if (metaMap != null && !getName().contentEquals(META_MAP_NAME)) {
+                return metaMap.get(getName());
+            }
         }
         return null;
     }
@@ -200,9 +216,11 @@ default Attributes getAttributes() {
      * Sets the attributes for this map.
      * */
     default void setAttributes(Attributes attributes) {
-        NitriteMap metaMap = getStore().openMap(META_MAP_NAME, String.class, Attributes.class);
-        if (metaMap != null && !getName().contentEquals(META_MAP_NAME)) {
-            metaMap.put(getName(), attributes);
+        if (!isDropped()) {
+            NitriteMap metaMap = getStore().openMap(META_MAP_NAME, String.class, Attributes.class);
+            if (metaMap != null && !getName().contentEquals(META_MAP_NAME)) {
+                metaMap.put(getName(), attributes);
+            }
         }
     }
 
@@ -210,17 +228,19 @@ default void setAttributes(Attributes attributes) {
      * Update last modified time of the map.
      */
     default void updateLastModifiedTime() {
-        if (isNullOrEmpty(getName())
-            || META_MAP_NAME.equals(getName())) return;
-
-        NitriteMap metaMap = getStore().openMap(META_MAP_NAME, String.class, Attributes.class);
-        if (metaMap != null) {
-            Attributes attributes = metaMap.get(getName());
-            if (attributes == null) {
-                attributes = new Attributes(getName());
-                metaMap.put(getName(), attributes);
+        if (!isDropped()) {
+            if (isNullOrEmpty(getName())
+                || META_MAP_NAME.equals(getName())) return;
+
+            NitriteMap metaMap = getStore().openMap(META_MAP_NAME, String.class, Attributes.class);
+            if (metaMap != null) {
+                Attributes attributes = metaMap.get(getName());
+                if (attributes == null) {
+                    attributes = new Attributes(getName());
+                    metaMap.put(getName(), attributes);
+                }
+                attributes.set(Attributes.LAST_MODIFIED_TIME, Long.toString(System.currentTimeMillis()));
             }
-            attributes.set(Attributes.LAST_MODIFIED_TIME, Long.toString(System.currentTimeMillis()));
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/NitriteStore.java b/nitrite/src/main/java/org/dizitart/no2/store/NitriteStore.java
index 1505c400e..a3cad2326 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/NitriteStore.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/NitriteStore.java
@@ -26,7 +26,7 @@
 import java.util.Set;
 
 /**
- * Represents a storage for Nitrite database.
+ * Represents a storage interface for Nitrite database.
  *
  * @param  the type parameter
  * @author Anindya Chatterjee
@@ -40,7 +40,7 @@ public interface NitriteStore extends NitritePlugin
     void openOrCreate();
 
     /**
-     * Checks whether this store is closed for further modification.
+     * Checks whether this store is closed.
      *
      * @return true if closed; false otherwise.
      */
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/StoreCatalog.java b/nitrite/src/main/java/org/dizitart/no2/store/StoreCatalog.java
index 106c8c849..d8b0a8fd3 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/StoreCatalog.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/StoreCatalog.java
@@ -59,11 +59,11 @@ public void writeCollectionEntry(String name) {
             document = Document.createDocument();
         }
 
-        // parse the document to create collection meta data object
+        // parse the document to create collection metadata object
         MapMetaData metaData = new MapMetaData(document);
         metaData.getMapNames().add(name);
 
-        // convert the meta data object to document and save
+        // convert the metadata object to document and save
         catalogMap.put(TAG_COLLECTIONS, metaData.getInfo());
     }
 
@@ -78,11 +78,11 @@ public void writeRepositoryEntry(String name) {
             document = Document.createDocument();
         }
 
-        // parse the document to create collection meta data object
+        // parse the document to create collection metadata object
         MapMetaData metaData = new MapMetaData(document);
         metaData.getMapNames().add(name);
 
-        // convert the meta data object to document and save
+        // convert the metadata object to document and save
         catalogMap.put(TAG_REPOSITORIES, metaData.getInfo());
     }
 
@@ -91,13 +91,13 @@ public void writeRepositoryEntry(String name) {
      *
      * @param name the name
      */
-    public void writeKeyedRepositoryEntries(String name) {
+    public void writeKeyedRepositoryEntry(String name) {
         Document document = catalogMap.get(TAG_KEYED_REPOSITORIES);
         if (document == null) {
             document = Document.createDocument();
         }
 
-        // parse the document to create collection meta data object
+        // parse the document to create collection metadata object
         MapMetaData metaData = new MapMetaData(document);
         metaData.getMapNames().add(name);
 
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/StoreMetaData.java b/nitrite/src/main/java/org/dizitart/no2/store/StoreMetaData.java
index 13181bc73..9bebc4622 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/StoreMetaData.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/StoreMetaData.java
@@ -44,7 +44,7 @@ public StoreMetaData(Document document) {
     }
 
     /**
-     * Gets the database info ina document.
+     * Gets the database info in a document.
      *
      * @return the info
      */
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/UserAuthenticationService.java b/nitrite/src/main/java/org/dizitart/no2/store/UserAuthenticationService.java
index 9286d86c8..5285c3485 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/UserAuthenticationService.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/UserAuthenticationService.java
@@ -57,9 +57,9 @@ public UserAuthenticationService(NitriteStore store) {
      *
      * @param username the username
      * @param password the password
-     * @param existing indicates if authentication data is already existing
      */
-    public void authenticate(String username, String password, boolean existing) {
+    public void authenticate(String username, String password) {
+        boolean existing = store.hasMap(USER_MAP);
         if (!isNullOrEmpty(password) && !isNullOrEmpty(username)) {
             if (!existing) {
                 byte[] salt = getNextSalt();
@@ -79,16 +79,14 @@ public void authenticate(String username, String password, boolean existing) {
                     byte[] expectedHash = userCredential.getPasswordHash();
 
                     if (notExpectedPassword(password.toCharArray(), salt, expectedHash)) {
-                        throw new NitriteSecurityException("username or password is invalid");
+                        throw new NitriteSecurityException("Username or password is invalid");
                     }
                 } else {
-                    throw new NitriteSecurityException("username or password is invalid");
+                    throw new NitriteSecurityException("Username or password is invalid");
                 }
             }
         } else if (existing) {
-            if (store.hasMap(USER_MAP)) {
-                throw new NitriteSecurityException("username or password is invalid");
-            }
+            throw new NitriteSecurityException("Username or password is invalid");
         }
     }
 
@@ -100,8 +98,7 @@ public void authenticate(String username, String password, boolean existing) {
      * @param oldPassword the old password
      * @param newPassword the new password
      */
-    public void addOrUpdatePassword(boolean update, String username,
-                                           SecureString oldPassword, SecureString newPassword) {
+    public void addOrUpdatePassword(boolean update, String username, SecureString oldPassword, SecureString newPassword) {
         NitriteMap userMap = null;
 
         if (update) {
@@ -113,12 +110,12 @@ public void addOrUpdatePassword(boolean update, String username,
                 byte[] expectedHash = credential.getPasswordHash();
 
                 if (notExpectedPassword(oldPassword.asString().toCharArray(), salt, expectedHash)) {
-                    throw new NitriteSecurityException("username or password is invalid");
+                    throw new NitriteSecurityException("Username or password is invalid");
                 }
             }
         } else {
             if (store.hasMap(USER_MAP)) {
-                throw new NitriteSecurityException("cannot add new credentials");
+                throw new NitriteSecurityException("Cannot add new credentials");
             }
         }
 
@@ -150,8 +147,7 @@ private byte[] hash(char[] password, byte[] salt) {
             return skf.generateSecret(spec).getEncoded();
         } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
             log.error("Error while hashing password", e);
-            throw new NitriteSecurityException("error while hashing a password: "
-                + e.getMessage());
+            throw new NitriteSecurityException("Error while hashing a password: " + e.getMessage());
         } finally {
             spec.clearPassword();
         }
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryMap.java b/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryMap.java
index 89590c6e9..f3323885d 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryMap.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryMap.java
@@ -3,6 +3,7 @@
 import org.dizitart.no2.common.RecordStream;
 import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.common.util.Comparables;
+import org.dizitart.no2.exceptions.InvalidOperationException;
 import org.dizitart.no2.store.NitriteMap;
 import org.dizitart.no2.store.NitriteStore;
 
@@ -47,11 +48,13 @@ public InMemoryMap(String mapName, NitriteStore nitriteStore) {
 
     @Override
     public boolean containsKey(Key key) {
+        checkOpened();
         return backingMap.containsKey(key);
     }
 
     @Override
     public Value get(Key key) {
+        checkOpened();
         return backingMap.get(key);
     }
 
@@ -60,12 +63,6 @@ public NitriteStore getStore() {
         return nitriteStore;
     }
 
-    @Override
-    public void clear() {
-        backingMap.clear();
-        updateLastModifiedTime();
-    }
-
     @Override
     public String getName() {
         return mapName;
@@ -73,11 +70,13 @@ public String getName() {
 
     @Override
     public RecordStream values() {
+        checkOpened();
         return RecordStream.fromIterable(backingMap.values());
     }
 
     @Override
     public Value remove(Key key) {
+        checkOpened();
         Value value = backingMap.remove(key);
         updateLastModifiedTime();
         return value;
@@ -85,11 +84,13 @@ public Value remove(Key key) {
 
     @Override
     public RecordStream keys() {
+        checkOpened();
         return RecordStream.fromIterable(backingMap.keySet());
     }
 
     @Override
     public void put(Key key, Value value) {
+        checkOpened();
         notNull(value, "value cannot be null");
         backingMap.put(key, value);
         updateLastModifiedTime();
@@ -97,33 +98,38 @@ public void put(Key key, Value value) {
 
     @Override
     public long size() {
+        checkOpened();
         return backingMap.size();
     }
 
     @Override
     public Value putIfAbsent(Key key, Value value) {
+        checkOpened();
         notNull(value, "value cannot be null");
 
         Value v = get(key);
         if (v == null) {
             put(key, value);
+            updateLastModifiedTime();
         }
-        updateLastModifiedTime();
         return v;
     }
 
     @Override
     public RecordStream> entries() {
+        checkOpened();
         return getStream(backingMap);
     }
 
     @Override
     public RecordStream> reversedEntries() {
+        checkOpened();
         return getStream(backingMap.descendingMap());
     }
 
     @Override
     public Key higherKey(Key key) {
+        checkOpened();
         if (key == null) {
             return null;
         }
@@ -132,6 +138,7 @@ public Key higherKey(Key key) {
 
     @Override
     public Key ceilingKey(Key key) {
+        checkOpened();
         if (key == null) {
             return null;
         }
@@ -140,6 +147,7 @@ public Key ceilingKey(Key key) {
 
     @Override
     public Key lowerKey(Key key) {
+        checkOpened();
         if (key == null) {
             return null;
         }
@@ -148,6 +156,7 @@ public Key lowerKey(Key key) {
 
     @Override
     public Key floorKey(Key key) {
+        checkOpened();
         if (key == null) {
             return null;
         }
@@ -156,23 +165,40 @@ public Key floorKey(Key key) {
 
     @Override
     public boolean isEmpty() {
+        checkOpened();
         return backingMap.isEmpty();
     }
 
     @Override
     public void drop() {
         if (!droppedFlag.get()) {
-            droppedFlag.compareAndSet(false, true);
-            clear();
+            backingMap.clear();
             getStore().removeMap(mapName);
+            droppedFlag.compareAndSet(false, true);
+            closedFlag.compareAndSet(false, true);
         }
     }
 
+    @Override
+    public boolean isDropped() {
+        return droppedFlag.get();
+    }
+
     @Override
     public void close() {
-        if (!closedFlag.get() && !droppedFlag.get()) {
-            closedFlag.compareAndSet(false, true);
-        }
+        closedFlag.compareAndSet(false, true);
+    }
+
+    @Override
+    public boolean isClosed() {
+        return closedFlag.get();
+    }
+
+    @Override
+    public void clear() {
+        checkOpened();
+        backingMap.clear();
+        updateLastModifiedTime();
     }
 
     private RecordStream> getStream(NavigableMap primaryMap) {
@@ -191,4 +217,14 @@ public Pair next() {
             }
         });
     }
+
+    private void checkOpened() {
+        if (closedFlag.get()) {
+            throw new InvalidOperationException("Map " + mapName + " is closed");
+        }
+
+        if (droppedFlag.get()) {
+            throw new InvalidOperationException("Map " + mapName + " is dropped");
+        }
+    }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryRTree.java b/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryRTree.java
index 293951224..ebedd2db1 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryRTree.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryRTree.java
@@ -2,14 +2,16 @@
 
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.RecordStream;
+import org.dizitart.no2.common.util.SpatialKey;
+import org.dizitart.no2.exceptions.InvalidOperationException;
 import org.dizitart.no2.index.BoundingBox;
 import org.dizitart.no2.store.NitriteRTree;
 
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * The in-memory {@link NitriteRTree}.
@@ -21,16 +23,21 @@
  */
 public class InMemoryRTree implements NitriteRTree {
     private final Map backingMap;
+    private final AtomicBoolean droppedFlag;
+    private final AtomicBoolean closedFlag;
 
     /**
      * Instantiates a new {@link InMemoryRTree}.
      */
     public InMemoryRTree() {
         this.backingMap = new ConcurrentHashMap<>();
+        this.closedFlag = new AtomicBoolean(false);
+        this.droppedFlag = new AtomicBoolean(false);
     }
 
     @Override
     public void add(Key key, NitriteId nitriteId) {
+        checkOpened();
         if (nitriteId != null && nitriteId.getIdValue() != null) {
             SpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue()));
             backingMap.put(spatialKey, key);
@@ -39,6 +46,7 @@ public void add(Key key, NitriteId nitriteId) {
 
     @Override
     public void remove(Key key, NitriteId nitriteId) {
+        checkOpened();
         if (nitriteId != null && nitriteId.getIdValue() != null) {
             SpatialKey spatialKey = getKey(key, Long.parseLong(nitriteId.getIdValue()));
             backingMap.remove(spatialKey);
@@ -47,6 +55,7 @@ public void remove(Key key, NitriteId nitriteId) {
 
     @Override
     public RecordStream findIntersectingKeys(Key key) {
+        checkOpened();
         SpatialKey spatialKey = getKey(key, 0L);
         Set set = new HashSet<>();
 
@@ -61,6 +70,7 @@ public RecordStream findIntersectingKeys(Key key) {
 
     @Override
     public RecordStream findContainedKeys(Key key) {
+        checkOpened();
         SpatialKey spatialKey = getKey(key, 0L);
         Set set = new HashSet<>();
 
@@ -73,159 +83,66 @@ public RecordStream findContainedKeys(Key key) {
         return RecordStream.fromIterable(set);
     }
 
-    private boolean isOverlap(SpatialKey a, SpatialKey b) {
-        if (a.isNull() || b.isNull()) {
-            return false;
-        }
-        for (int i = 0; i < 2; i++) {
-            if (a.max(i) < b.min(i) || a.min(i) > b.max(i)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    private boolean isInside(SpatialKey a, SpatialKey b) {
-        if (a.isNull() || b.isNull()) {
-            return false;
-        }
-        for (int i = 0; i < 2; i++) {
-            if (a.min(i) <= b.min(i) || a.max(i) >= b.max(i)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
     @Override
     public long size() {
+        checkOpened();
         return backingMap.size();
     }
 
-    private SpatialKey getKey(Key key, long id) {
-        return new SpatialKey(id, key.getMinX(),
-            key.getMaxX(), key.getMinY(), key.getMaxY());
-    }
-
     @Override
     public void close() {
-
+        closedFlag.compareAndSet(false, true);
     }
 
     @Override
     public void clear() {
+        checkOpened();
         backingMap.clear();
     }
 
     @Override
     public void drop() {
+        checkOpened();
+        droppedFlag.compareAndSet(false, true);
         backingMap.clear();
     }
 
-    /**
-     * The type Spatial key.
-     */
-    static class SpatialKey {
-
-        private final long id;
-        private final float[] minMax;
-
-        /**
-         * Instantiates a new Spatial key.
-         *
-         * @param id     the id
-         * @param minMax the min max
-         */
-        public SpatialKey(long id, float... minMax) {
-            this.id = id;
-            this.minMax = minMax;
-        }
-
-        /**
-         * Min float.
-         *
-         * @param dim the dim
-         * @return the float
-         */
-        public float min(int dim) {
-            return minMax[dim + dim];
-        }
-
-        /**
-         * Sets min.
-         *
-         * @param dim the dim
-         * @param x   the x
-         */
-        public void setMin(int dim, float x) {
-            minMax[dim + dim] = x;
-        }
-
-        /**
-         * Max float.
-         *
-         * @param dim the dim
-         * @return the float
-         */
-        public float max(int dim) {
-            return minMax[dim + dim + 1];
-        }
-
-        /**
-         * Sets max.
-         *
-         * @param dim the dim
-         * @param x   the x
-         */
-        public void setMax(int dim, float x) {
-            minMax[dim + dim + 1] = x;
-        }
+    private SpatialKey getKey(Key key, long id) {
+        return new SpatialKey(id, key.getMinX(),
+            key.getMaxX(), key.getMinY(), key.getMaxY());
+    }
 
-        /**
-         * Gets id.
-         *
-         * @return the id
-         */
-        public long getId() {
-            return id;
+    private boolean isOverlap(SpatialKey a, SpatialKey b) {
+        if (a.isNull() || b.isNull()) {
+            return false;
         }
-
-        /**
-         * Is null boolean.
-         *
-         * @return the boolean
-         */
-        public boolean isNull() {
-            return minMax.length == 0;
+        for (int i = 0; i < 2; i++) {
+            if (a.max(i) < b.min(i) || a.min(i) > b.max(i)) {
+                return false;
+            }
         }
+        return true;
+    }
 
-        @Override
-        public int hashCode() {
-            return (int) ((id >>> 32) ^ id);
+    private boolean isInside(SpatialKey a, SpatialKey b) {
+        if (a.isNull() || b.isNull()) {
+            return false;
         }
-
-        @Override
-        public boolean equals(Object other) {
-            if (other == this) {
-                return true;
-            } else if (!(other instanceof SpatialKey)) {
-                return false;
-            }
-            SpatialKey o = (SpatialKey) other;
-            if (id != o.id) {
+        for (int i = 0; i < 2; i++) {
+            if (a.min(i) <= b.min(i) || a.max(i) >= b.max(i)) {
                 return false;
             }
-            return equalsIgnoringId(o);
+        }
+        return true;
+    }
+
+    private void checkOpened() {
+        if (closedFlag.get()) {
+            throw new InvalidOperationException("RTreeMap is closed");
         }
 
-        /**
-         * Equals ignoring id boolean.
-         *
-         * @param o the o
-         * @return the boolean
-         */
-        public boolean equalsIgnoringId(SpatialKey o) {
-            return Arrays.equals(minMax, o.minMax);
+        if (droppedFlag.get()) {
+            throw new InvalidOperationException("RTreeMap is dropped");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryStore.java b/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryStore.java
index 1568dee3e..992fd0127 100644
--- a/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryStore.java
+++ b/nitrite/src/main/java/org/dizitart/no2/store/memory/InMemoryStore.java
@@ -85,7 +85,12 @@ public boolean hasMap(String mapName) {
     @SuppressWarnings("unchecked")
     public  NitriteMap openMap(String mapName, Class keyType, Class valueType) {
         if (nitriteMapRegistry.containsKey(mapName)) {
-            return (InMemoryMap) nitriteMapRegistry.get(mapName);
+            NitriteMap nitriteMap = (NitriteMap) nitriteMapRegistry.get(mapName);
+            if (nitriteMap.isClosed()) {
+                nitriteMapRegistry.remove(mapName);
+            } else {
+                return nitriteMap;
+            }
         }
 
         NitriteMap nitriteMap = new InMemoryMap<>(mapName, this);
@@ -97,19 +102,23 @@ public  NitriteMap openMap(String mapName, Class keyT
     @Override
     public void closeMap(String mapName) {
         // nothing to close as it is volatile map, moreover,
-        // removing it form registry means loosing the map
+        // removing it from registry means losing the map
     }
 
     @Override
     public void closeRTree(String rTreeName) {
         // nothing to close as it is volatile map, moreover,
-        // removing it form registry means loosing the map
+        // removing it from registry means losing the map
     }
 
     @Override
     public void removeMap(String mapName) {
         if (nitriteMapRegistry.containsKey(mapName)) {
-            nitriteMapRegistry.get(mapName).clear();
+            NitriteMap nitriteMap = nitriteMapRegistry.get(mapName);
+            if (!nitriteMap.isClosed() && !nitriteMap.isDropped()) {
+                nitriteMap.clear();
+                nitriteMap.close();
+            }
             nitriteMapRegistry.remove(mapName);
             getCatalog().remove(mapName);
         }
@@ -138,7 +147,7 @@ public String getStoreVersion() {
     private void initEventBus() {
         if (getStoreConfig().eventListeners() != null) {
             for (StoreEventListener eventListener : getStoreConfig().eventListeners()) {
-                eventBus.register(eventListener);
+                subscribe(eventListener);
             }
         }
     }
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/ChangeType.java b/nitrite/src/main/java/org/dizitart/no2/transaction/ChangeType.java
index 6c4ab7323..d7c296ed1 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/ChangeType.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/ChangeType.java
@@ -1,3 +1,20 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package org.dizitart.no2.transaction;
 
 /**
@@ -23,7 +40,7 @@ enum ChangeType {
     Remove,
 
     /**
-     * Clear.
+     * Clear. Commit only operation, cannot be rolled back.
      */
     Clear,
 
@@ -45,25 +62,15 @@ enum ChangeType {
     /**
      * Drop all indices.
      */
-    DropAllIndices,
+    DropAllIndexes,
 
     /**
-     * Drop collection.
+     * Drop collection. Commit only operation, cannot be rolled back.
      */
     DropCollection,
 
     /**
      * Set attribute.
      */
-    SetAttribute,
-
-    /**
-     * Add processor
-     * */
-    AddProcessor,
-
-    /**
-     * Remove processor
-     * */
-    RemoveProcessor,
+    SetAttributes,
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalCollection.java b/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalCollection.java
index 30b01c4ac..c2fe7af27 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalCollection.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalCollection.java
@@ -3,17 +3,16 @@
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.Setter;
-import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.NitriteConfig;
 import org.dizitart.no2.collection.*;
 import org.dizitart.no2.collection.events.CollectionEventInfo;
 import org.dizitart.no2.collection.events.CollectionEventListener;
-import org.dizitart.no2.collection.meta.Attributes;
 import org.dizitart.no2.collection.operation.CollectionOperations;
 import org.dizitart.no2.common.Fields;
 import org.dizitart.no2.common.WriteResult;
 import org.dizitart.no2.common.event.EventBus;
 import org.dizitart.no2.common.event.NitriteEventBus;
+import org.dizitart.no2.common.meta.Attributes;
 import org.dizitart.no2.common.processors.Processor;
 import org.dizitart.no2.exceptions.*;
 import org.dizitart.no2.filters.Filter;
@@ -45,8 +44,6 @@
 class DefaultTransactionalCollection implements NitriteCollection {
     private final NitriteCollection primary;
     private final TransactionContext transactionContext;
-    private final Nitrite nitrite;
-
     private String collectionName;
     private NitriteMap nitriteMap;
     private NitriteStore nitriteStore;
@@ -67,11 +64,9 @@ class DefaultTransactionalCollection implements NitriteCollection {
     private EventBus, CollectionEventListener> eventBus;
 
     public DefaultTransactionalCollection(NitriteCollection primary,
-                                          TransactionContext transactionContext,
-                                          Nitrite nitrite) {
+                                          TransactionContext transactionContext) {
         this.primary = primary;
         this.transactionContext = transactionContext;
-        this.nitrite = nitrite;
 
         initialize();
     }
@@ -159,14 +154,14 @@ public WriteResult update(Document document, boolean insertIfAbsent) {
             if (document.hasId()) {
                 return update(createUniqueFilter(document), document, updateOptions(false));
             } else {
-                throw new NotIdentifiableException("update operation failed as no id value found for the document");
+                throw new NotIdentifiableException("Update operation failed as no id value found for the document");
             }
         }
     }
 
     @Override
     public WriteResult remove(Document document) {
-        notNull(document, "a null document cannot be removed");
+        notNull(document, "A null document cannot be removed");
 
         WriteResult result;
         if (document.hasId()) {
@@ -178,7 +173,7 @@ public WriteResult remove(Document document) {
                 writeLock.unlock();
             }
         } else {
-            throw new NotIdentifiableException("remove operation failed as no id value found for the document");
+            throw new NotIdentifiableException("Remove operation failed as no id value found for the document");
         }
 
         AtomicReference toRemove = new AtomicReference<>();
@@ -202,7 +197,7 @@ public WriteResult remove(Document document) {
     @Override
     public WriteResult remove(Filter filter, boolean justOne) {
         if ((filter == null || filter == Filter.ALL) && justOne) {
-            throw new InvalidOperationException("remove all cannot be combined with just once");
+            throw new InvalidOperationException("Remove all cannot be combined with just once");
         }
 
         WriteResult result;
@@ -279,30 +274,6 @@ public void addProcessor(Processor processor) {
         } finally {
             writeLock.unlock();
         }
-
-        JournalEntry journalEntry = new JournalEntry();
-        journalEntry.setChangeType(ChangeType.AddProcessor);
-        journalEntry.setCommit(() -> primary.addProcessor(processor));
-        journalEntry.setRollback(() -> primary.removeProcessor(processor));
-        transactionContext.getJournal().add(journalEntry);
-    }
-
-    @Override
-    public void removeProcessor(Processor processor) {
-        notNull(processor, "a null processor cannot be removed");
-        try {
-            writeLock.lock();
-            checkOpened();
-            collectionOperations.addProcessor(processor);
-        } finally {
-            writeLock.unlock();
-        }
-
-        JournalEntry journalEntry = new JournalEntry();
-        journalEntry.setChangeType(ChangeType.RemoveProcessor);
-        journalEntry.setCommit(() -> primary.removeProcessor(processor));
-        journalEntry.setRollback(() -> primary.addProcessor(processor));
-        transactionContext.getJournal().add(journalEntry);
     }
 
     @Override
@@ -423,7 +394,7 @@ public void dropIndex(String... fieldNames) {
         journalEntry.setChangeType(ChangeType.DropIndex);
         journalEntry.setCommit(() -> {
             for (IndexDescriptor entry : primary.listIndices()) {
-                if (entry.getIndexFields().equals(fields)) {
+                if (entry.getFields().equals(fields)) {
                     indexEntry.set(entry);
                     break;
                 }
@@ -451,14 +422,14 @@ public void dropAllIndices() {
         List indexEntries = new ArrayList<>();
 
         JournalEntry journalEntry = new JournalEntry();
-        journalEntry.setChangeType(ChangeType.DropAllIndices);
+        journalEntry.setChangeType(ChangeType.DropAllIndexes);
         journalEntry.setCommit(() -> {
             indexEntries.addAll(primary.listIndices());
             primary.dropAllIndices();
         });
         journalEntry.setRollback(() -> {
             for (IndexDescriptor indexDescriptor : indexEntries) {
-                String[] fieldNames = indexDescriptor.getIndexFields().getFieldNames().toArray(new String[0]);
+                String[] fieldNames = indexDescriptor.getFields().getFieldNames().toArray(new String[0]);
                 primary.createIndex(indexOptions(indexDescriptor.getIndexType()), fieldNames);
             }
         });
@@ -470,24 +441,15 @@ public void clear() {
         try {
             writeLock.lock();
             checkOpened();
-            nitriteMap.clear();
+            collectionOperations.clear();
         } finally {
             writeLock.unlock();
         }
 
-        List documentList = new ArrayList<>();
-
         JournalEntry journalEntry = new JournalEntry();
         journalEntry.setChangeType(ChangeType.Clear);
-        journalEntry.setCommit(() -> {
-            documentList.addAll(primary.find().toList());
-            primary.clear();
-        });
-        journalEntry.setRollback(() -> {
-            for (Document document : documentList) {
-                primary.insert(document);
-            }
-        });
+        journalEntry.setCommit(primary::clear);
+        journalEntry.setRollback(() -> {}); // can't be rolled back
         transactionContext.getJournal().add(journalEntry);
     }
 
@@ -502,28 +464,10 @@ public void drop() {
         }
         isDropped = true;
 
-        List documentList = new ArrayList<>();
-        List indexEntries = new ArrayList<>();
-
         JournalEntry journalEntry = new JournalEntry();
         journalEntry.setChangeType(ChangeType.DropCollection);
-        journalEntry.setCommit(() -> {
-            documentList.addAll(primary.find().toList());
-            indexEntries.addAll(primary.listIndices());
-            primary.drop();
-        });
-        journalEntry.setRollback(() -> {
-            NitriteCollection collection = nitrite.getCollection(collectionName);
-
-            for (IndexDescriptor indexDescriptor : indexEntries) {
-                String[] fieldNames = indexDescriptor.getIndexFields().getFieldNames().toArray(new String[0]);
-                collection.createIndex(indexOptions(indexDescriptor.getIndexType()), fieldNames);
-            }
-
-            for (Document document : documentList) {
-                collection.insert(document);
-            }
-        });
+        journalEntry.setCommit(primary::drop);
+        journalEntry.setRollback(() -> {}); // can't be rolled back
         transactionContext.getJournal().add(journalEntry);
     }
 
@@ -538,7 +482,7 @@ public boolean isOpen() {
             try {
                 close();
             } catch (Exception e) {
-                throw new NitriteIOException("failed to close the database", e);
+                throw new NitriteIOException("Failed to close the collection", e);
             }
             return false;
         } else return true;
@@ -615,7 +559,7 @@ public void setAttributes(Attributes attributes) {
         AtomicReference original = new AtomicReference<>();
 
         JournalEntry journalEntry = new JournalEntry();
-        journalEntry.setChangeType(ChangeType.SetAttribute);
+        journalEntry.setChangeType(ChangeType.SetAttributes);
         journalEntry.setCommit(() -> {
             original.set(primary.getAttributes());
             primary.setAttributes(attributes);
@@ -654,19 +598,19 @@ public void post(CollectionEventInfo collectionEventInfo) {
 
     private void checkOpened() {
         if (isClosed) {
-            throw new TransactionException("collection is closed");
+            throw new TransactionException("Collection is closed");
         }
 
         if (!primary.isOpen()) {
-            throw new NitriteIOException("store is closed");
+            throw new TransactionException("Store is closed");
         }
 
         if (isDropped()) {
-            throw new NitriteIOException("collection has been dropped");
+            throw new TransactionException("Collection is dropped");
         }
 
         if (!transactionContext.getActive().get()) {
-            throw new TransactionException("transaction is closed");
+            throw new TransactionException("Transaction is not active");
         }
     }
 
@@ -678,11 +622,11 @@ private void closeEventBus() {
     }
 
     private void validateRebuildIndex(IndexDescriptor indexDescriptor) {
-        notNull(indexDescriptor, "indexEntry cannot be null");
+        notNull(indexDescriptor, "indexDescriptor cannot be null");
 
-        String[] fieldNames = indexDescriptor.getIndexFields().getFieldNames().toArray(new String[0]);
+        String[] fieldNames = indexDescriptor.getFields().getFieldNames().toArray(new String[0]);
         if (isIndexing(fieldNames)) {
-            throw new IndexingException("indexing on value " + indexDescriptor.getIndexFields() + " is currently running");
+            throw new IndexingException("Indexing on fields " + indexDescriptor.getFields() + " is currently running");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalRepository.java b/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalRepository.java
index 54bb8c62a..d2de7972a 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalRepository.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/DefaultTransactionalRepository.java
@@ -4,17 +4,19 @@
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.FindOptions;
 import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.collection.UpdateOptions;
 import org.dizitart.no2.collection.events.CollectionEventListener;
-import org.dizitart.no2.collection.meta.Attributes;
 import org.dizitart.no2.common.WriteResult;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.meta.Attributes;
+import org.dizitart.no2.common.processors.Processor;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.index.IndexDescriptor;
 import org.dizitart.no2.index.IndexOptions;
-import org.dizitart.no2.common.mapper.NitriteMapper;
-import org.dizitart.no2.common.processors.Processor;
 import org.dizitart.no2.repository.Cursor;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.RepositoryOperations;
+import org.dizitart.no2.repository.EntityDecorator;
 import org.dizitart.no2.store.NitriteStore;
 
 import java.util.Collection;
@@ -28,11 +30,12 @@
  * @since 4.0
  */
 class DefaultTransactionalRepository implements ObjectRepository {
-    private final Class type;
     private final ObjectRepository primary;
     private final NitriteCollection backingCollection;
     private final NitriteConfig nitriteConfig;
     private RepositoryOperations operations;
+    private Class type;
+    private EntityDecorator entityDecorator;
 
     public DefaultTransactionalRepository(Class type,
                                           ObjectRepository primary,
@@ -45,14 +48,20 @@ public DefaultTransactionalRepository(Class type,
         initialize();
     }
 
-    @Override
-    public void addProcessor(Processor processor) {
-        backingCollection.addProcessor(processor);
+    public DefaultTransactionalRepository(EntityDecorator entityDecorator,
+                                          ObjectRepository primary,
+                                          NitriteCollection backingCollection,
+                                          TransactionConfig nitriteConfig) {
+        this.entityDecorator = entityDecorator;
+        this.primary = primary;
+        this.backingCollection = backingCollection;
+        this.nitriteConfig = nitriteConfig;
+        initialize();
     }
 
     @Override
-    public void removeProcessor(Processor processor) {
-        backingCollection.removeProcessor(processor);
+    public void addProcessor(Processor processor) {
+        backingCollection.addProcessor(processor);
     }
 
     @Override
@@ -102,19 +111,19 @@ public WriteResult insert(T[] elements) {
     public WriteResult update(T element, boolean insertIfAbsent) {
         notNull(element, "a null object cannot be used for update");
 
-        return update(operations.createUniqueFilter(element), element, insertIfAbsent);
+        return update(operations.createUniqueFilter(element), element, updateOptions(insertIfAbsent, true));
     }
 
     @Override
-    public WriteResult update(Filter filter, T update, boolean insertIfAbsent) {
+    public WriteResult update(Filter filter, T update, UpdateOptions updateOptions) {
         notNull(update, "a null object cannot be used for update");
 
         Document updateDocument = operations.toDocument(update, true);
-        if (!insertIfAbsent) {
+        if (updateOptions == null || !updateOptions.isInsertIfAbsent()) {
             operations.removeNitriteId(updateDocument);
         }
         return backingCollection.update(operations.asObjectFilter(filter), updateDocument,
-            updateOptions(insertIfAbsent, true));
+            updateOptions);
     }
 
     @Override
@@ -123,7 +132,8 @@ public WriteResult update(Filter filter, Document update, boolean justOnce) {
         operations.removeNitriteId(update);
         operations.serializeFields(update);
 
-        return backingCollection.update(operations.asObjectFilter(filter), update, updateOptions(false, justOnce));
+        return backingCollection.update(operations.asObjectFilter(filter), update,
+            updateOptions(false, justOnce));
     }
 
     @Override
@@ -209,7 +219,11 @@ public void setAttributes(Attributes attributes) {
 
     @Override
     public Class getType() {
-        return type;
+        if (entityDecorator != null) {
+            return entityDecorator.getEntityType();
+        } else {
+            return type;
+        }
     }
 
     @Override
@@ -219,7 +233,11 @@ public NitriteCollection getDocumentCollection() {
 
     private void initialize() {
         NitriteMapper nitriteMapper = nitriteConfig.nitriteMapper();
-        this.operations = new RepositoryOperations(type, nitriteMapper, backingCollection);
+        if (entityDecorator != null) {
+            this.operations = new RepositoryOperations(entityDecorator, nitriteMapper, backingCollection);
+        } else {
+            this.operations = new RepositoryOperations(type, nitriteMapper, backingCollection);
+        }
         this.operations.createIndices();
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/NitriteTransaction.java b/nitrite/src/main/java/org/dizitart/no2/transaction/NitriteTransaction.java
index 26af4fbc3..ae3d8e94a 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/NitriteTransaction.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/NitriteTransaction.java
@@ -11,6 +11,7 @@
 import org.dizitart.no2.common.module.NitriteModule;
 import org.dizitart.no2.exceptions.TransactionException;
 import org.dizitart.no2.repository.ObjectRepository;
+import org.dizitart.no2.repository.EntityDecorator;
 import org.dizitart.no2.store.NitriteMap;
 import org.dizitart.no2.store.NitriteStore;
 
@@ -19,6 +20,7 @@
 import java.util.concurrent.locks.Lock;
 
 import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryName;
+import static org.dizitart.no2.common.util.ObjectUtils.findRepositoryNameByDecorator;
 
 /**
  * @author Anindya Chatterjee
@@ -39,7 +41,7 @@ class NitriteTransaction implements Transaction {
     @Getter
     private String id;
 
-    private State state;
+    private TransactionState state;
 
     public NitriteTransaction(Nitrite nitrite, LockService lockService) {
         this.nitrite = nitrite;
@@ -59,7 +61,7 @@ public synchronized NitriteCollection getCollection(String name) {
         if (nitrite.hasCollection(name)) {
             primary = nitrite.getCollection(name);
         } else {
-            throw new TransactionException("collection " + name + " does not exists");
+            throw new TransactionException("Collection " + name + " does not exists");
         }
 
         NitriteMap txMap = transactionStore.openMap(name,
@@ -71,7 +73,7 @@ public synchronized NitriteCollection getCollection(String name) {
         context.setJournal(new LinkedList<>());
         context.setConfig(transactionConfig);
 
-        NitriteCollection txCollection = new DefaultTransactionalCollection(primary, context, nitrite);
+        NitriteCollection txCollection = new DefaultTransactionalCollection(primary, context);
         collectionRegistry.put(name, txCollection);
         contextMap.put(name, context);
         return txCollection;
@@ -91,7 +93,7 @@ public synchronized  ObjectRepository getRepository(Class type) {
         if (nitrite.hasRepository(type)) {
             primary = nitrite.getRepository(type);
         } else {
-            throw new TransactionException("repository of type " + type.getName() + " does not exists");
+            throw new TransactionException("Repository of type " + type.getName() + " does not exists");
         }
 
         NitriteMap txMap = transactionStore.openMap(name,
@@ -104,7 +106,7 @@ public synchronized  ObjectRepository getRepository(Class type) {
         context.setConfig(transactionConfig);
 
         NitriteCollection primaryCollection = primary.getDocumentCollection();
-        NitriteCollection backingCollection = new DefaultTransactionalCollection(primaryCollection, context, nitrite);
+        NitriteCollection backingCollection = new DefaultTransactionalCollection(primaryCollection, context);
         ObjectRepository txRepository = new DefaultTransactionalRepository<>(type,
             primary, backingCollection, transactionConfig);
 
@@ -127,7 +129,7 @@ public synchronized  ObjectRepository getRepository(Class type, String
         if (nitrite.hasRepository(type, key)) {
             primary = nitrite.getRepository(type, key);
         } else {
-            throw new TransactionException("repository of type " + type.getName()
+            throw new TransactionException("Repository of type " + type.getName()
                 + " and key " + key + " does not exists");
         }
 
@@ -141,7 +143,7 @@ public synchronized  ObjectRepository getRepository(Class type, String
         context.setConfig(transactionConfig);
 
         NitriteCollection primaryCollection = primary.getDocumentCollection();
-        NitriteCollection backingCollection = new DefaultTransactionalCollection(primaryCollection, context, nitrite);
+        NitriteCollection backingCollection = new DefaultTransactionalCollection(primaryCollection, context);
         ObjectRepository txRepository = new DefaultTransactionalRepository<>(type,
             primary, backingCollection, transactionConfig);
         repositoryRegistry.put(name, txRepository);
@@ -149,10 +151,82 @@ public synchronized  ObjectRepository getRepository(Class type, String
         return txRepository;
     }
 
+    @Override
+    @SuppressWarnings("unchecked")
+    public synchronized  ObjectRepository getRepository(EntityDecorator entityDecorator) {
+        checkState();
+
+        String name = findRepositoryNameByDecorator(entityDecorator, null);
+        if (repositoryRegistry.containsKey(name)) {
+            return (ObjectRepository) repositoryRegistry.get(name);
+        }
+
+        ObjectRepository primary;
+        if (nitrite.hasRepository(entityDecorator)) {
+            primary = nitrite.getRepository(entityDecorator);
+        } else {
+            throw new TransactionException("Repository of type " + entityDecorator.getEntityName() + " does not exists");
+        }
+
+        NitriteMap txMap = transactionStore.openMap(name,
+            NitriteId.class, Document.class);
+
+        TransactionContext context = new TransactionContext();
+        context.setCollectionName(name);
+        context.setNitriteMap(txMap);
+        context.setJournal(new LinkedList<>());
+        context.setConfig(transactionConfig);
+
+        NitriteCollection primaryCollection = primary.getDocumentCollection();
+        NitriteCollection backingCollection = new DefaultTransactionalCollection(primaryCollection, context);
+        ObjectRepository txRepository = new DefaultTransactionalRepository<>(entityDecorator,
+            primary, backingCollection, transactionConfig);
+
+        repositoryRegistry.put(name, txRepository);
+        contextMap.put(name, context);
+        return txRepository;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public synchronized  ObjectRepository getRepository(EntityDecorator entityDecorator, String key) {
+        checkState();
+
+        String name = findRepositoryNameByDecorator(entityDecorator, key);
+        if (repositoryRegistry.containsKey(name)) {
+            return (ObjectRepository) repositoryRegistry.get(name);
+        }
+
+        ObjectRepository primary;
+        if (nitrite.hasRepository(entityDecorator, key)) {
+            primary = nitrite.getRepository(entityDecorator, key);
+        } else {
+            throw new TransactionException("Repository of type " + entityDecorator.getEntityName()
+                + " and key " + key + " does not exists");
+        }
+
+        NitriteMap txMap = transactionStore.openMap(name,
+            NitriteId.class, Document.class);
+
+        TransactionContext context = new TransactionContext();
+        context.setCollectionName(name);
+        context.setNitriteMap(txMap);
+        context.setJournal(new LinkedList<>());
+        context.setConfig(transactionConfig);
+
+        NitriteCollection primaryCollection = primary.getDocumentCollection();
+        NitriteCollection backingCollection = new DefaultTransactionalCollection(primaryCollection, context);
+        ObjectRepository txRepository = new DefaultTransactionalRepository<>(entityDecorator,
+            primary, backingCollection, transactionConfig);
+        repositoryRegistry.put(name, txRepository);
+        contextMap.put(name, context);
+        return txRepository;
+    }
+
     @Override
     public synchronized void commit() {
         checkState();
-        this.state = State.PartiallyCommitted;
+        this.state = TransactionState.PartiallyCommitted;
 
         for (Map.Entry contextEntry : contextMap.entrySet()) {
             String collectionName = contextEntry.getKey();
@@ -184,13 +258,13 @@ public synchronized void commit() {
                     }
                 }
             } catch (TransactionException te) {
-                state = State.Failed;
+                state = TransactionState.Failed;
                 log.error("Error while committing transaction", te);
                 throw te;
             } catch (Exception e) {
-                state = State.Failed;
+                state = TransactionState.Failed;
                 log.error("Error while committing transaction", e);
-                throw new TransactionException("failed to commit transaction", e);
+                throw new TransactionException("Error committing transaction", e);
             } finally {
                 undoRegistry.put(collectionName, undoLog);
                 transactionContext.getActive().set(false);
@@ -198,13 +272,13 @@ public synchronized void commit() {
             }
         }
 
-        state = State.Committed;
+        state = TransactionState.Committed;
         close();
     }
 
     @Override
     public synchronized void rollback() {
-        this.state = State.Aborted;
+        this.state = TransactionState.Aborted;
 
         for (Map.Entry> entry : undoRegistry.entrySet()) {
             String collectionName = entry.getKey();
@@ -233,7 +307,7 @@ public synchronized void rollback() {
     @Override
     public synchronized void close() {
         try {
-            state = State.Closed;
+            state = TransactionState.Closed;
             for (TransactionContext context : contextMap.values()) {
                 context.getActive().set(false);
             }
@@ -245,12 +319,11 @@ public synchronized void close() {
             this.transactionStore.close();
             this.transactionConfig.close();
         } catch (Exception e) {
-            throw new TransactionException("transaction failed to close", e);
+            throw new TransactionException("Error closing transaction", e);
         }
     }
 
-    @Override
-    public synchronized State getState() {
+    public synchronized TransactionState getState() {
         return state;
     }
 
@@ -270,12 +343,12 @@ private void prepare() {
         this.transactionConfig.autoConfigure();
         this.transactionConfig.initialize();
         this.transactionStore = (TransactionStore) this.transactionConfig.getNitriteStore();
-        this.state = State.Active;
+        this.state = TransactionState.Active;
     }
 
     private void checkState() {
-        if (state != State.Active) {
-            throw new TransactionException("transaction is not active");
+        if (state != TransactionState.Active) {
+            throw new TransactionException("Transaction is not active");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/Session.java b/nitrite/src/main/java/org/dizitart/no2/transaction/Session.java
index 7a8c0a93c..22bf76ae5 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/Session.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/Session.java
@@ -57,7 +57,7 @@ public Transaction beginTransaction() {
     public void close() {
         this.active.compareAndSet(true, false);
         for (Transaction transaction : transactionMap.values()) {
-            if (transaction.getState() != State.Closed) {
+            if (transaction.getState() != TransactionState.Closed) {
                 transaction.rollback();
             }
         }
@@ -72,7 +72,7 @@ public void close() {
      */
     public void checkState() {
         if (!active.get()) {
-            throw new TransactionException("this session is not active");
+            throw new TransactionException("Session is closed");
         }
     }
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/Transaction.java b/nitrite/src/main/java/org/dizitart/no2/transaction/Transaction.java
index 5986c6cc7..ffc156a72 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/Transaction.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/Transaction.java
@@ -2,6 +2,7 @@
 
 import org.dizitart.no2.collection.NitriteCollection;
 import org.dizitart.no2.repository.ObjectRepository;
+import org.dizitart.no2.repository.EntityDecorator;
 
 /**
  * Represents an ACID transaction on nitrite database.
@@ -22,7 +23,7 @@ public interface Transaction extends AutoCloseable {
      *
      * @return the state
      */
-    State getState();
+    TransactionState getState();
 
     /**
      * Gets a {@link NitriteCollection} to perform ACID operations on it.
@@ -51,6 +52,26 @@ public interface Transaction extends AutoCloseable {
      */
      ObjectRepository getRepository(Class type, String key);
 
+
+    /**
+     * Gets an {@link ObjectRepository} to perform ACID operations on it.
+     *
+     * @param   the type parameter
+     * @param entityDecorator the entityDecorator
+     * @return the repository
+     */
+     ObjectRepository getRepository(EntityDecorator entityDecorator);
+
+    /**
+     * Gets an {@link ObjectRepository} to perform ACID operations on it.
+     *
+     * @param   the type parameter
+     * @param entityDecorator the entityDecorator
+     * @param key  the key
+     * @return the repository
+     */
+     ObjectRepository getRepository(EntityDecorator entityDecorator, String key);
+
     /**
      * Completes the transaction and commits the data to the underlying store.
      */
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionConfig.java b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionConfig.java
index 476033470..3e8e6a979 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionConfig.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionConfig.java
@@ -2,11 +2,7 @@
 
 import lombok.extern.slf4j.Slf4j;
 import org.dizitart.no2.NitriteConfig;
-import org.dizitart.no2.exceptions.IndexingException;
-import org.dizitart.no2.index.NitriteIndexer;
 import org.dizitart.no2.common.mapper.NitriteMapper;
-import org.dizitart.no2.common.module.NitriteModule;
-import org.dizitart.no2.store.NitriteStore;
 
 /**
  * @author Anindya Chatterjee
@@ -21,43 +17,16 @@ public TransactionConfig(NitriteConfig config) {
         this.config = config;
     }
 
-    @Override
-    public NitriteIndexer findIndexer(String indexType) {
-        NitriteIndexer nitriteIndexer = pluginManager.getIndexerMap().get(indexType);
-        if (nitriteIndexer != null) {
-            nitriteIndexer.initialize(this);
-            return nitriteIndexer;
-        } else {
-            throw new IndexingException("no indexer found for index type " + indexType);
-        }
-    }
-
     @Override
     public void fieldSeparator(String separator) {
         config.fieldSeparator(separator);
     }
 
-    @Override
-    public NitriteConfig loadModule(NitriteModule module) {
-        pluginManager.loadModule(module);
-        return this;
-    }
-
-    @Override
-    public void autoConfigure() {
-        pluginManager.findAndLoadPlugins();
-    }
-
     @Override
     public NitriteMapper nitriteMapper() {
         return config.nitriteMapper();
     }
 
-    @Override
-    public NitriteStore getNitriteStore() {
-        return pluginManager.getNitriteStore();
-    }
-
     @Override
     public void initialize() {
         super.initialize();
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/State.java b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionState.java
similarity index 63%
rename from nitrite/src/main/java/org/dizitart/no2/transaction/State.java
rename to nitrite/src/main/java/org/dizitart/no2/transaction/TransactionState.java
index 39bad9902..88129c048 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/State.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionState.java
@@ -6,34 +6,34 @@
  * @author Anindya Chatterjee
  * @since 4.0
  */
-public enum State {
+public enum TransactionState {
     /**
      * Transaction is active.
      */
     Active,
 
     /**
-     * Transaction is partially committed.
+     * Transaction partially committed.
      */
     PartiallyCommitted,
 
     /**
-     * Transaction is fully committed.
+     * Transaction fully committed.
      */
     Committed,
 
     /**
-     * Transaction is closed.
+     * Transaction closed.
      */
     Closed,
 
     /**
-     * Transaction is failed.
+     * Transaction failed.
      */
     Failed,
 
     /**
-     * Transaction is aborted.
+     * Transaction aborted.
      */
     Aborted,
 }
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionStore.java b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionStore.java
index 95ba2c7aa..b1e19a377 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionStore.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionStore.java
@@ -45,7 +45,7 @@ public boolean isReadOnly() {
 
     @Override
     public void commit() {
-        throw new InvalidOperationException("call commit on transaction");
+        throw new InvalidOperationException("Call commit on transaction");
     }
 
     @Override
@@ -79,7 +79,12 @@ public boolean hasMap(String mapName) {
     @SuppressWarnings("unchecked")
     public  NitriteMap openMap(String mapName, Class keyType, Class valueType) {
         if (mapRegistry.containsKey(mapName)) {
-            return (NitriteMap) mapRegistry.get(mapName);
+            NitriteMap nitriteMap = (NitriteMap) mapRegistry.get(mapName);
+            if (nitriteMap.isClosed()) {
+                mapRegistry.remove(mapName);
+            } else {
+                return nitriteMap;
+            }
         }
 
         NitriteMap primaryMap = null;
@@ -95,13 +100,13 @@ public  NitriteMap openMap(String mapName, Class keyT
     @Override
     public void closeMap(String mapName) {
         // nothing to close as it is volatile map, moreover,
-        // removing it form registry means loosing the map
+        // removing it from registry means losing the map
     }
 
     @Override
     public void closeRTree(String rTreeName) {
         // nothing to close as it is volatile map, moreover,
-        // removing it form registry means loosing the map
+        // removing it from registry means losing the map
     }
 
     @Override
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalMap.java b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalMap.java
index b3053c000..8aa7b9657 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalMap.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalMap.java
@@ -64,7 +64,7 @@ public V get(K k) {
         if (result == null) {
             result = primary.get(k);
             if (result instanceof CopyOnWriteArrayList) {
-                // create a deep copy of the list so that it does not effect the original one
+                // create a deep copy of the list so that it does not affect the original one
                 List list = deepCopy((CopyOnWriteArrayList) result);
                 backingMap.put(k, (V) list);
                 result = (V) list;
@@ -284,22 +284,29 @@ public boolean isEmpty() {
     @Override
     public void drop() {
         if (!droppedFlag.get()) {
-            droppedFlag.compareAndSet(false, true);
-            closedFlag.compareAndSet(false, true);
-
-            cleared = true;
             backingMap.clear();
             tombstones.clear();
+            cleared = true;
+            droppedFlag.compareAndSet(false, true);
         }
     }
 
+    @Override
+    public boolean isDropped() {
+        return droppedFlag.get();
+    }
+
     @Override
     public void close() {
-        if (!closedFlag.get() && !droppedFlag.get()) {
-            closedFlag.compareAndSet(false, true);
-            backingMap.clear();
-            tombstones.clear();
-        }
+        backingMap.clear();
+        tombstones.clear();
+        cleared = true;
+        closedFlag.compareAndSet(false, true);
+    }
+
+    @Override
+    public boolean isClosed() {
+        return closedFlag.get();
     }
 
     private RecordStream> getStream(RecordStream> primaryStream,
diff --git a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalRTree.java b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalRTree.java
index 1b96895e2..6c077b42b 100644
--- a/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalRTree.java
+++ b/nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalRTree.java
@@ -2,10 +2,14 @@
 
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.RecordStream;
+import org.dizitart.no2.common.util.SpatialKey;
 import org.dizitart.no2.index.BoundingBox;
 import org.dizitart.no2.store.NitriteRTree;
 
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * @author Anindya Chatterjee
@@ -114,67 +118,4 @@ public void clear() {
     public void drop() {
         map.clear();
     }
-
-    /*
-     * Copyright 2004-2019 H2 Group. Multiple-Licensed under the MPL 2.0,
-     * and the EPL 1.0 (https://h2database.com/html/license.html).
-     * Initial Developer: H2 Group
-     */
-    private static class SpatialKey {
-
-        private final long id;
-        private final float[] minMax;
-
-        public SpatialKey(long id, float... minMax) {
-            this.id = id;
-            this.minMax = minMax;
-        }
-
-        public float min(int dim) {
-            return minMax[dim + dim];
-        }
-
-        public void setMin(int dim, float x) {
-            minMax[dim + dim] = x;
-        }
-
-        public float max(int dim) {
-            return minMax[dim + dim + 1];
-        }
-
-        public void setMax(int dim, float x) {
-            minMax[dim + dim + 1] = x;
-        }
-
-        public long getId() {
-            return id;
-        }
-
-        public boolean isNull() {
-            return minMax.length == 0;
-        }
-
-        @Override
-        public int hashCode() {
-            return (int) ((id >>> 32) ^ id);
-        }
-
-        @Override
-        public boolean equals(Object other) {
-            if (other == this) {
-                return true;
-            } else if (!(other instanceof SpatialKey)) {
-                return false;
-            }
-            SpatialKey o = (SpatialKey) other;
-            if (id != o.id) {
-                return false;
-            }
-            return equalsIgnoringId(o);
-        }
-
-        public boolean equalsIgnoringId(SpatialKey o) {
-            return Arrays.equals(minMax, o.minMax);
-        }
-    }
 }
diff --git a/nitrite/src/main/resources/version b/nitrite/src/main/resources/version
index cc868b62c..99eba4de9 100644
--- a/nitrite/src/main/resources/version
+++ b/nitrite/src/main/resources/version
@@ -1 +1 @@
-4.0.1
\ No newline at end of file
+4.1.0
\ No newline at end of file
diff --git a/nitrite/src/test/java/org/dizitart/no2/NitriteBuilderTest.java b/nitrite/src/test/java/org/dizitart/no2/NitriteBuilderTest.java
index 59d877268..b9da9bf68 100644
--- a/nitrite/src/test/java/org/dizitart/no2/NitriteBuilderTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/NitriteBuilderTest.java
@@ -24,6 +24,7 @@
 import org.dizitart.no2.common.FieldValues;
 import org.dizitart.no2.common.Fields;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.common.module.NitriteModule;
 import org.dizitart.no2.common.module.NitritePlugin;
 import org.dizitart.no2.common.module.PluginManager;
@@ -117,11 +118,11 @@ public void testAddMigrations() {
     public void testAddMigrations2() {
         NitriteBuilder builderResult = Nitrite.builder();
         Migration migration = mock(Migration.class);
-        when(migration.getEndVersion()).thenReturn(1);
-        when(migration.getStartVersion()).thenReturn(1);
+        when(migration.getToVersion()).thenReturn(1);
+        when(migration.getFromVersion()).thenReturn(1);
         assertSame(builderResult, builderResult.addMigrations(migration, null, null));
-        verify(migration).getEndVersion();
-        verify(migration).getStartVersion();
+        verify(migration).getToVersion();
+        verify(migration).getFromVersion();
     }
 
     @Test
@@ -138,7 +139,7 @@ public void testOpenOrCreate() {
         assertFalse(actualOpenOrCreateResult.isClosed());
         assertFalse(store.isClosed());
         assertSame(store, pluginManager.getNitriteStore());
-        assertTrue(pluginManager.getNitriteMapper() instanceof org.dizitart.no2.common.mapper.MappableMapper);
+        assertTrue(pluginManager.getNitriteMapper() instanceof SimpleDocumentMapper);
     }
 
     @Test
@@ -152,7 +153,7 @@ public void testOpenOrCreate2() {
         assertFalse(store.isClosed());
         PluginManager pluginManager = config.getPluginManager();
         assertEquals(3, pluginManager.getIndexerMap().size());
-        assertTrue(pluginManager.getNitriteMapper() instanceof org.dizitart.no2.common.mapper.MappableMapper);
+        assertTrue(pluginManager.getNitriteMapper() instanceof SimpleDocumentMapper);
         assertTrue(store.getCatalog().getKeyedRepositoryNames().isEmpty());
         assertSame(store, pluginManager.getNitriteStore());
         assertTrue(((InMemoryConfig) store.getStoreConfig()).eventListeners().isEmpty());
@@ -169,7 +170,7 @@ public void testOpenOrCreate3() {
         assertFalse(store.isClosed());
         PluginManager pluginManager = config.getPluginManager();
         assertEquals(3, pluginManager.getIndexerMap().size());
-        assertTrue(pluginManager.getNitriteMapper() instanceof org.dizitart.no2.common.mapper.MappableMapper);
+        assertTrue(pluginManager.getNitriteMapper() instanceof SimpleDocumentMapper);
         assertTrue(store.getCatalog().getKeyedRepositoryNames().isEmpty());
         assertSame(store, pluginManager.getNitriteStore());
         assertTrue(((InMemoryConfig) store.getStoreConfig()).eventListeners().isEmpty());
@@ -184,7 +185,7 @@ public void testOpenOrCreate4() {
         assertEquals(3, pluginManager.getIndexerMap().size());
         NitriteStore nitriteStore = nitriteConfig.getNitriteStore();
         assertSame(nitriteStore, pluginManager.getNitriteStore());
-        assertTrue(pluginManager.getNitriteMapper() instanceof org.dizitart.no2.common.mapper.MappableMapper);
+        assertTrue(pluginManager.getNitriteMapper() instanceof SimpleDocumentMapper);
         assertFalse(nitriteStore.isClosed());
         assertTrue(((InMemoryConfig) nitriteStore.getStoreConfig()).eventListeners().isEmpty());
     }
@@ -198,7 +199,7 @@ public void testOpenOrCreate5() {
         NitriteStore nitriteStore = nitriteConfig.getNitriteStore();
         assertFalse(nitriteStore.isClosed());
         assertSame(nitriteStore, pluginManager.getNitriteStore());
-        assertTrue(pluginManager.getNitriteMapper() instanceof org.dizitart.no2.common.mapper.MappableMapper);
+        assertTrue(pluginManager.getNitriteMapper() instanceof SimpleDocumentMapper);
     }
 
     @Test
@@ -312,16 +313,6 @@ public  Target convert(Source source, Class type) {
             return null;
         }
 
-        @Override
-        public boolean isValueType(Class type) {
-            return false;
-        }
-
-        @Override
-        public boolean isValue(Object object) {
-            return false;
-        }
-
         @Override
         public void initialize(NitriteConfig nitriteConfig) {
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/NitriteConfigTest.java b/nitrite/src/test/java/org/dizitart/no2/NitriteConfigTest.java
index 684d4840e..eb8bc3d55 100644
--- a/nitrite/src/test/java/org/dizitart/no2/NitriteConfigTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/NitriteConfigTest.java
@@ -17,6 +17,7 @@
 
 package org.dizitart.no2;
 
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.common.module.NitriteModule;
 import org.dizitart.no2.common.module.NitritePlugin;
 import org.dizitart.no2.common.module.PluginManager;
@@ -47,12 +48,6 @@ public void testConstructor() {
 
     @Test
     public void testFieldSeparator() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
         (new NitriteConfig()).fieldSeparator("Separator");
     }
 
@@ -75,39 +70,39 @@ public void testAddMigration() {
     public void testAddMigration2() {
         NitriteConfig nitriteConfig = new NitriteConfig();
         Migration migration = mock(Migration.class);
-        when(migration.getEndVersion()).thenReturn(1);
-        when(migration.getStartVersion()).thenReturn(1);
+        when(migration.getToVersion()).thenReturn(1);
+        when(migration.getFromVersion()).thenReturn(1);
         assertSame(nitriteConfig, nitriteConfig.addMigration(migration));
-        verify(migration).getEndVersion();
-        verify(migration).getStartVersion();
+        verify(migration).getToVersion();
+        verify(migration).getFromVersion();
     }
 
     @Test
     public void testAddMigration3() {
         NitriteConfig nitriteConfig = new NitriteConfig();
         Migration migration = mock(Migration.class);
-        when(migration.getEndVersion()).thenReturn(4);
-        when(migration.getStartVersion()).thenReturn(1);
+        when(migration.getToVersion()).thenReturn(4);
+        when(migration.getFromVersion()).thenReturn(1);
         assertSame(nitriteConfig, nitriteConfig.addMigration(migration));
-        verify(migration).getEndVersion();
-        verify(migration).getStartVersion();
+        verify(migration).getToVersion();
+        verify(migration).getFromVersion();
     }
 
     @Test
     public void testAddMigration4() {
         NitriteConfig nitriteConfig = new NitriteConfig();
         Migration migration = mock(Migration.class);
-        when(migration.getEndVersion()).thenReturn(1);
-        when(migration.getStartVersion()).thenReturn(4);
+        when(migration.getToVersion()).thenReturn(1);
+        when(migration.getFromVersion()).thenReturn(4);
         assertSame(nitriteConfig, nitriteConfig.addMigration(migration));
-        verify(migration).getEndVersion();
-        verify(migration).getStartVersion();
+        verify(migration).getToVersion();
+        verify(migration).getFromVersion();
     }
 
     @Test
     public void testSchemaVersion() {
         NitriteConfig nitriteConfig = new NitriteConfig();
-        NitriteConfig actualSchemaVersionResult = nitriteConfig.schemaVersion(1);
+        NitriteConfig actualSchemaVersionResult = nitriteConfig.currentSchemaVersion(1);
         assertSame(nitriteConfig, actualSchemaVersionResult);
         assertEquals(1, actualSchemaVersionResult.getSchemaVersion().intValue());
     }
@@ -120,7 +115,7 @@ public void testAutoConfigure() {
         assertEquals(3, pluginManager.getIndexerMap().size());
         NitriteStore nitriteStore = nitriteConfig.getNitriteStore();
         assertSame(nitriteStore, pluginManager.getNitriteStore());
-        assertTrue(pluginManager.getNitriteMapper() instanceof org.dizitart.no2.common.mapper.MappableMapper);
+        assertTrue(pluginManager.getNitriteMapper() instanceof SimpleDocumentMapper);
         assertFalse(nitriteStore.isClosed());
         assertTrue(((InMemoryConfig) nitriteStore.getStoreConfig()).eventListeners().isEmpty());
     }
diff --git a/nitrite/src/test/java/org/dizitart/no2/NitriteDatabaseTest.java b/nitrite/src/test/java/org/dizitart/no2/NitriteDatabaseTest.java
index ea64480e6..0611d5590 100644
--- a/nitrite/src/test/java/org/dizitart/no2/NitriteDatabaseTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/NitriteDatabaseTest.java
@@ -17,48 +17,18 @@
 
 package org.dizitart.no2;
 
-import org.dizitart.no2.exceptions.NitriteIOException;
-import org.dizitart.no2.exceptions.NitriteSecurityException;
 import org.dizitart.no2.store.memory.InMemoryStoreModule;
 import org.junit.Test;
 
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 public class NitriteDatabaseTest {
     @Test
     public void testConstructor() {
         NitriteConfig nitriteConfig = new NitriteConfig();
         nitriteConfig.loadModule(new InMemoryStoreModule());
-        new NitriteDatabase("janedoe", "iloveyou", nitriteConfig);
-        assertTrue(nitriteConfig.configured);
-    }
-
-    @Test
-    public void testConstructor2() {
-        assertThrows(NitriteSecurityException.class, () -> new NitriteDatabase("", "iloveyou", new NitriteConfig()));
-    }
-
-    @Test
-    public void testConstructor3() {
-        assertThrows(NitriteSecurityException.class, () -> new NitriteDatabase("janedoe", "", new NitriteConfig()));
-    }
-
-    @Test
-    public void testConstructor4() {
-        assertThrows(NitriteIOException.class, () -> new NitriteDatabase("janedoe", "iloveyou", null));
-    }
-
-    @Test(expected = NitriteIOException.class)
-    public void testConstructor5() {
-        NitriteConfig nitriteConfig = new NitriteConfig();
         new NitriteDatabase(nitriteConfig);
-        assertTrue(nitriteConfig.configured);
-    }
-
-    @Test
-    public void testConstructor6() {
-        assertThrows(NitriteIOException.class, () -> new NitriteDatabase(null));
+        assertFalse(nitriteConfig.configured);
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/DocumentTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/DocumentTest.java
index 772d71d73..195621c25 100644
--- a/nitrite/src/test/java/org/dizitart/no2/collection/DocumentTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/collection/DocumentTest.java
@@ -18,7 +18,6 @@
 package org.dizitart.no2.collection;
 
 import org.dizitart.no2.NitriteConfig;
-import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.exceptions.InvalidIdException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.junit.After;
@@ -26,12 +25,15 @@
 import org.junit.Test;
 
 import java.io.*;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
-import static org.dizitart.no2.integration.TestUtil.parse;
 import static org.dizitart.no2.collection.Document.createDocument;
 import static org.dizitart.no2.common.Constants.DOC_ID;
 import static org.dizitart.no2.common.util.Iterables.listOf;
+import static org.dizitart.no2.integration.TestUtil.parse;
 import static org.junit.Assert.*;
 
 public class DocumentTest {
@@ -171,15 +173,35 @@ public void testGet() {
 
     @Test
     public void testRemove() {
-        Iterator> iterator = doc.iterator();
         assertEquals(doc.size(), 4);
-        if (iterator.hasNext()) {
-            iterator.next();
-            iterator.remove();
-        }
+        doc.remove("score");
         assertEquals(doc.size(), 3);
     }
 
+    @Test
+    public void testRemoveWithCustomFieldSeparator() {
+        NitriteConfig config = new NitriteConfig();
+        config.fieldSeparator(":");
+        assertEquals(((Document)doc.get("location:address")).size(), 3);
+        doc.remove("location:address:line1");
+        assertEquals(((Document)doc.get("location:address")).size(), 2);
+        config.fieldSeparator(".");
+    }
+    @Test
+    public void testRemoveFromArray() {
+        NitriteConfig config = new NitriteConfig();
+        config.fieldSeparator(":");
+
+        assertEquals(((List)doc.get("location:address:house")).size(), 3);
+        doc.remove("location:address:house:0");
+        assertEquals(((List)doc.get("location:address:house")).size(), 2);
+
+        assertEquals(((List) doc.get("objArray")).size(), 2);
+        doc.remove("objArray:0:value");
+        assertEquals(((List) doc.get("objArray")).size(), 2);
+        assertEquals(((Document) doc.get("objArray:0")).size(), 0);
+    }
+
     @Test
     public void getFields() {
         Set fields = doc.getFields();
@@ -193,7 +215,7 @@ public void getFields() {
 
     @Test
     @SuppressWarnings("unchecked")
-    public void getEmbeddedArrayFields() {
+    public void testGetEmbeddedArrayFields() {
         Document document = createDocument("first", "value")
             .put("seconds", new String[]{"1", "2"})
             .put("third", null)
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/FindOptionsTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/FindOptionsTest.java
index db71d7ff3..57698e77b 100644
--- a/nitrite/src/test/java/org/dizitart/no2/collection/FindOptionsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/collection/FindOptionsTest.java
@@ -121,5 +121,13 @@ public void testThenOrderBy2() {
         FindOptions orderByResult = FindOptions.orderBy("Field Name", SortOrder.Ascending);
         assertSame(orderByResult, orderByResult.thenOrderBy("Field Name", SortOrder.Ascending));
     }
+
+    @Test
+    public void testDistinct() {
+        FindOptions findOptions = new FindOptions();
+        FindOptions actualDistinctResult = findOptions.withDistinct(true);
+        assertSame(findOptions, actualDistinctResult);
+        assertTrue(actualDistinctResult.distinct());
+    }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/FindPlanTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/FindPlanTest.java
index b18f506d2..2d79cb372 100644
--- a/nitrite/src/test/java/org/dizitart/no2/collection/FindPlanTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/collection/FindPlanTest.java
@@ -31,7 +31,8 @@ public void testConstructor() {
         assertTrue(actualFindPlan.getBlockingSortOrder().isEmpty());
         assertEquals(
             "FindPlan(byIdFilter=null, indexScanFilter=null, collectionScanFilter=null, indexDescriptor=null,"
-                + " indexScanOrder=null, blockingSortOrder=[], skip=null, limit=null, collator=null, subPlans=[])",
+                + " indexScanOrder=null, blockingSortOrder=[], skip=null, limit=null, distinct=false, " +
+                "collator=null, subPlans=[])",
             actualFindPlan.toString());
         assertTrue(actualFindPlan.getSubPlans().isEmpty());
         assertNull(actualFindPlan.getSkip());
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/NitriteCollectionTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/NitriteCollectionTest.java
index 37ec3a5dd..915803581 100644
--- a/nitrite/src/test/java/org/dizitart/no2/collection/NitriteCollectionTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/collection/NitriteCollectionTest.java
@@ -18,7 +18,7 @@
 
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.integration.Retry;
-import org.dizitart.no2.collection.meta.Attributes;
+import org.dizitart.no2.common.meta.Attributes;
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/SnowflakeIdGeneratorTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/SnowflakeIdGeneratorTest.java
index 5c0273aa4..659a2cc23 100644
--- a/nitrite/src/test/java/org/dizitart/no2/collection/SnowflakeIdGeneratorTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/collection/SnowflakeIdGeneratorTest.java
@@ -19,6 +19,10 @@
 
 import org.junit.Test;
 
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class SnowflakeIdGeneratorTest {
@@ -32,5 +36,16 @@ public void testTillNextMillis() {
     public void testGetId() {
         assertTrue((new SnowflakeIdGenerator()).getId() != 0);
     }
+
+    @Test
+    public void testUniqueness() {
+        SnowflakeIdGenerator generator = new SnowflakeIdGenerator();
+        Set ids = new HashSet<>();
+        for (int j = 0; j < 100; j++) {
+            ids.add(generator.getId());
+        }
+
+        assertEquals(100, ids.size());
+    }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/meta/MetadataAwareTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/meta/MetadataAwareTest.java
deleted file mode 100644
index 5fdfb97bf..000000000
--- a/nitrite/src/test/java/org/dizitart/no2/collection/meta/MetadataAwareTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.dizitart.no2.collection.meta;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.junit.Assert.assertNull;
-
-public class MetadataAwareTest {
-    @Test
-    public void testGetAttributes() {
-        MetadataAware metadataAware = new MetadataAware() {
-            @Override
-            public Attributes getAttributes() {
-                return new Attributes();
-            }
-
-            @Override
-            public void setAttributes(Attributes attributes) {
-
-            }
-        };
-        Assert.assertNotNull("This is a boilerplate assert on the result.", metadataAware.getAttributes());
-    }
-
-    @Test
-    public void testSetAttributes() {
-        MetadataAware metadataAware = new MetadataAware() {
-            @Override
-            public Attributes getAttributes() {
-                return null;
-            }
-
-            @Override
-            public void setAttributes(Attributes attributes) {
-                assertNull(attributes);
-            }
-        };
-        metadataAware.setAttributes(null);
-    }
-}
-
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/operation/ReadOperationsTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/operation/ReadOperationsTest.java
index bbb99823d..0d5aef3a7 100644
--- a/nitrite/src/test/java/org/dizitart/no2/collection/operation/ReadOperationsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/collection/operation/ReadOperationsTest.java
@@ -27,6 +27,7 @@
 import org.dizitart.no2.index.IndexDescriptor;
 import org.dizitart.no2.store.memory.InMemoryMap;
 import org.junit.Test;
+import org.mockito.internal.verification.NoInteractions;
 
 import java.util.ArrayList;
 
@@ -68,12 +69,16 @@ public void testFind2() {
 
     @Test
     public void testGetById() {
+        IndexOperations indexOperations = mock(IndexOperations.class);
+        when(indexOperations.listIndexes()).thenReturn(new ArrayList<>());
+
         NitriteConfig nitriteConfig = new NitriteConfig();
         InMemoryMap nitriteMap = new InMemoryMap<>("Map Name", null);
 
-        ReadOperations readOperations = new ReadOperations("Collection Name", null, nitriteConfig, nitriteMap,
+        ReadOperations readOperations = new ReadOperations("Collection Name", indexOperations, nitriteConfig, nitriteMap,
                 new ProcessorChain());
         assertNull(readOperations.getById(NitriteId.newId()));
+        verify(indexOperations, new NoInteractions()).listIndexes();
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/FieldsTest.java b/nitrite/src/test/java/org/dizitart/no2/common/FieldsTest.java
index 34887c467..6bb9b9cc3 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/FieldsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/FieldsTest.java
@@ -17,6 +17,7 @@
 
 package org.dizitart.no2.common;
 
+import org.dizitart.no2.exceptions.ValidationException;
 import org.junit.Test;
 
 import java.util.List;
@@ -55,7 +56,7 @@ public void testStartsWith() {
         assertTrue(fields.startsWith(new Fields()));
     }
 
-    @Test
+    @Test(expected = ValidationException.class)
     public void testStartsWith2() {
         assertFalse((new Fields()).startsWith(null));
     }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/event/EventTest.java b/nitrite/src/test/java/org/dizitart/no2/common/event/EventTest.java
index bf73a45bb..cd274e7c1 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/event/EventTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/event/EventTest.java
@@ -18,6 +18,8 @@
 
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.NitriteBuilder;
+import org.dizitart.no2.collection.UpdateOptions;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.collection.events.EventType;
 import org.dizitart.no2.repository.ObjectRepository;
@@ -74,6 +76,9 @@ public void setUp() {
             db = nitriteBuilder.openOrCreate();
         }
 
+        SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        documentMapper.registerEntityConverter(new Employee.EmployeeConverter());
+
         employeeRepository = db.getRepository(Employee.class);
         listener = new SampleListenerCollection();
         employeeRepository.subscribe(listener);
@@ -115,7 +120,7 @@ public void testUpsert() {
         e.setEmpId(1L);
         e.setAddress("abcd");
 
-        employeeRepository.update(where("empId").eq(1), e, true);
+        employeeRepository.update(where("empId").eq(1), e, UpdateOptions.updateOptions(true));
         await().atMost(1, TimeUnit.SECONDS).until(listenerPrepared(EventType.Insert));
         assertEquals(listener.getAction(), EventType.Insert);
         assertNotNull(listener.getItem());
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/Company.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/Company.java
index ad947db76..3e99586d3 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/mapper/Company.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/mapper/Company.java
@@ -25,27 +25,13 @@
  * @author Anindya Chatterjee.
  */
 @Data
-public class Company implements Mappable {
+public class Company {
     private String name;
     private Long id;
     private CompanyId companyId;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("id", id)
-            .put("name", name)
-            .put("companyId", mapper.convert(companyId, Document.class));
-    }
-
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        id = document.get("id", Long.class);
-        companyId = document.get("companyId", CompanyId.class);
-    }
-
     @Data
-    public static class CompanyId implements Comparable, Serializable, Mappable {
+    public static class CompanyId implements Comparable, Serializable {
         private Long idValue;
 
         public CompanyId(long value) {
@@ -57,14 +43,47 @@ public int compareTo(CompanyId other) {
             return idValue.compareTo(other.idValue);
         }
 
+        public static class CompanyIdConverter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return CompanyId.class;
+            }
+
+            @Override
+            public Document toDocument(CompanyId entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument().put("idValue", entity.idValue);
+            }
+
+            @Override
+            public CompanyId fromDocument(Document document, NitriteMapper nitriteMapper) {
+                CompanyId entity = new CompanyId(0L);
+                entity.idValue = document.get("idValue", Long.class);
+                return entity;
+            }
+        }
+    }
+
+    public static class CompanyConverter implements EntityConverter {
+        @Override
+        public Class getEntityType() {
+            return Company.class;
+        }
+
         @Override
-        public Document write(NitriteMapper mapper) {
-            return Document.createDocument().put("idValue", idValue);
+        public Document toDocument(Company entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("id", entity.id)
+                .put("name", entity.name)
+                .put("companyId", nitriteMapper.convert(entity.companyId, Document.class));
         }
 
         @Override
-        public void read(NitriteMapper mapper, Document document) {
-            idValue = document.get("idValue", Long.class);
+        public Company fromDocument(Document document, NitriteMapper nitriteMapper) {
+            Company entity = new Company();
+            entity.name = document.get("name", String.class);
+            entity.id = document.get("id", Long.class);
+            entity.companyId = nitriteMapper.convert(document.get("companyId", Document.class), CompanyId.class);
+            return entity;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/Department.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/Department.java
index 5d1a5eb59..5c83a37fe 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/mapper/Department.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/mapper/Department.java
@@ -28,32 +28,42 @@
  */
 @Data
 @ToString
-public class Department implements Mappable {
+public class Department {
     private String name;
     private List employeeList;
 
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        List docList = new ArrayList<>();
-        if (employeeList != null && !employeeList.isEmpty()) {
-            employeeList.stream().map(employee -> mapper.convert(employee, Document.class))
-                .forEach(docList::add);
+    public static class DepartmentConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return Department.class;
         }
 
-        return Document.createDocument().put("name", name)
-            .put("employeeList", docList);
-    }
+        @Override
+        public Document toDocument(Department entity, NitriteMapper nitriteMapper) {
+            List docList = new ArrayList<>();
+            if (entity.employeeList != null && !entity.employeeList.isEmpty()) {
+                entity.employeeList.stream().map(employee -> nitriteMapper.convert(employee, Document.class))
+                    .forEach(docList::add);
+            }
+
+            return Document.createDocument().put("name", entity.name)
+                .put("employeeList", docList);
+        }
 
-    @Override
-    @SuppressWarnings("unchecked")
-    public void read(NitriteMapper mapper, Document document) {
-        employeeList = new ArrayList<>();
-        List documentList = (List) document.get("employeeList", ArrayList.class);
-        if (documentList != null && !documentList.isEmpty()) {
-            documentList.stream().map(doc -> mapper.convert(doc, MappableEmployee.class))
-                .forEach(employeeList::add);
+        @Override
+        @SuppressWarnings("unchecked")
+        public Department fromDocument(Document document, NitriteMapper nitriteMapper) {
+            Department entity = new Department();
+            entity.employeeList = new ArrayList<>();
+            List documentList = (List) document.get("employeeList", ArrayList.class);
+            if (documentList != null && !documentList.isEmpty()) {
+                documentList.stream().map(doc -> nitriteMapper.convert(doc, MappableEmployee.class))
+                    .forEach(entity.employeeList::add);
+            }
+            entity.name = document.get("name", String.class);
+            return entity;
         }
-        name = document.get("name", String.class);
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/Employee.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/Employee.java
index 057a0f0cb..3d76fb41f 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/mapper/Employee.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/mapper/Employee.java
@@ -27,34 +27,44 @@
  */
 @Data
 @ToString
-public class Employee implements Mappable {
+public class Employee {
     private String empId;
     private String name;
     private Date joiningDate;
     private Employee boss;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        Document document = Document.createDocument("empId", getEmpId())
-            .put("name", getName())
-            .put("joiningDate", getJoiningDate());
+    public static class Converter implements EntityConverter {
 
-        if (getBoss() != null) {
-            document.put("boss", mapper.convert(getBoss(), Document.class));
+        @Override
+        public Class getEntityType() {
+            return Employee.class;
+        }
+
+        @Override
+        public Document toDocument(Employee entity, NitriteMapper nitriteMapper) {
+            Document document = Document.createDocument("empId", entity.getEmpId())
+                .put("name", entity.getName())
+                .put("joiningDate", entity.getJoiningDate());
+
+            if (entity.getBoss() != null) {
+                document.put("boss", nitriteMapper.convert(entity.getBoss(), Document.class));
+            }
+            return document;
         }
-        return document;
-    }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        setEmpId(document.get("empId", String.class));
-        setName(document.get("name", String.class));
-        setJoiningDate(document.get("joiningDate", Date.class));
+        @Override
+        public Employee fromDocument(Document document, NitriteMapper nitriteMapper) {
+            Employee entity = new Employee();
+            entity.setEmpId(document.get("empId", String.class));
+            entity.setName(document.get("name", String.class));
+            entity.setJoiningDate(document.get("joiningDate", Date.class));
 
-        Document bossDoc = document.get("boss", Document.class);
-        if (bossDoc != null) {
-            Employee boss = mapper.convert(bossDoc, Employee.class);
-            setBoss(boss);
+            Document bossDoc = document.get("boss", Document.class);
+            if (bossDoc != null) {
+                Employee boss = nitriteMapper.convert(bossDoc, Employee.class);
+                entity.setBoss(boss);
+            }
+            return entity;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableDepartment.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableDepartment.java
index b61548d0f..31dd66714 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableDepartment.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableDepartment.java
@@ -28,41 +28,50 @@
  */
 @Data
 @ToString
-public class MappableDepartment implements Mappable {
+public class MappableDepartment {
     private String name;
     private List employeeList;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        Document document = Document.createDocument();
 
-        document.put("name", getName());
-        List employees = new ArrayList<>();
-        for (MappableEmployee employee : getEmployeeList()) {
-            employees.add(employee.write(mapper));
+    private List getEmployeeList() {
+        if (employeeList == null) {
+            employeeList = new ArrayList<>();
         }
-        document.put("employeeList", employees);
-
-        return document;
+        return employeeList;
     }
 
-    @Override
-    @SuppressWarnings("unchecked")
-    public void read(NitriteMapper mapper, Document document) {
-        if (document != null) {
-            setName((String) document.get("name"));
-            for (Document doc : (List) document.get("employeeList")) {
-                MappableEmployee me = new MappableEmployee();
-                me.read(mapper, doc);
-                getEmployeeList().add(me);
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return MappableDepartment.class;
+        }
+
+        @Override
+        public Document toDocument(MappableDepartment entity, NitriteMapper nitriteMapper) {
+            Document document = Document.createDocument();
+
+            document.put("name", entity.getName());
+            List employees = new ArrayList<>();
+            for (MappableEmployee employee : entity.getEmployeeList()) {
+                employees.add(nitriteMapper.convert(employee, Document.class));
             }
+            document.put("employeeList", employees);
+
+            return document;
         }
-    }
 
-    private List getEmployeeList() {
-        if (employeeList == null) {
-            employeeList = new ArrayList<>();
+        @Override
+        public MappableDepartment fromDocument(Document document, NitriteMapper nitriteMapper) {
+            MappableDepartment entity = new MappableDepartment();
+            if (document != null) {
+                entity.setName((String) document.get("name"));
+                for (Document doc : (List) document.get("employeeList")) {
+                    MappableEmployee me = nitriteMapper.convert(doc, MappableEmployee.class);
+                    entity.getEmployeeList().add(me);
+                }
+            }
+            return entity;
         }
-        return employeeList;
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableEmployee.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableEmployee.java
index a2d7f79a1..dd0d01e90 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableEmployee.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableEmployee.java
@@ -27,39 +27,50 @@
  */
 @Data
 @ToString
-public class MappableEmployee implements Mappable {
+public class MappableEmployee {
     private String empId;
     private String name;
     private Date joiningDate;
     private MappableEmployee boss;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        Document document = Document.createDocument();
-        document.put("empId", getEmpId());
-        document.put("name", getName());
-        document.put("joiningDate", getJoiningDate());
+    public static class MappableEmployeeConverter implements EntityConverter {
 
-        if (getBoss() != null) {
-            Document bossDoc = getBoss().write(mapper);
-            document.put("boss", bossDoc);
+        @Override
+        public Class getEntityType() {
+            return MappableEmployee.class;
         }
-        return document;
-    }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        if (document != null) {
-            setEmpId((String) document.get("empId"));
-            setName((String) document.get("name"));
-            setJoiningDate((Date) document.get("joiningDate"));
+        @Override
+        public Document toDocument(MappableEmployee entity, NitriteMapper nitriteMapper) {
+            Document document = Document.createDocument();
+            document.put("empId", entity.getEmpId());
+            document.put("name", entity.getName());
+            document.put("joiningDate", entity.getJoiningDate());
 
-            Document bossDoc = (Document) document.get("boss");
-            if (bossDoc != null) {
-                MappableEmployee bossEmp = new MappableEmployee();
-                bossEmp.read(mapper, bossDoc);
-                setBoss(bossEmp);
+            if (entity.getBoss() != null) {
+                Document bossDoc = nitriteMapper.convert(entity.getBoss(), Document.class);
+                document.put("boss", bossDoc);
             }
+            return document;
+        }
+
+        @Override
+        public MappableEmployee fromDocument(Document document, NitriteMapper nitriteMapper) {
+            MappableEmployee entity = new MappableEmployee();
+
+            if (document != null) {
+                entity.setEmpId((String) document.get("empId"));
+                entity.setName((String) document.get("name"));
+                entity.setJoiningDate((Date) document.get("joiningDate"));
+
+                Document bossDoc = (Document) document.get("boss");
+                if (bossDoc != null) {
+                    MappableEmployee bossEmp = nitriteMapper.convert(bossDoc, MappableEmployee.class);
+                    entity.setBoss(bossEmp);
+                }
+            }
+
+            return entity;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableMapperTest.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableMapperTest.java
deleted file mode 100644
index 1e64953db..000000000
--- a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MappableMapperTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-package org.dizitart.no2.common.mapper;
-
-import org.dizitart.no2.integration.NitriteStressTest;
-import org.dizitart.no2.integration.NitriteTest;
-import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.exceptions.ObjectMappingException;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class MappableMapperTest {
-    @Test
-    public void testConvert() {
-        MappableMapper mappableMapper = new MappableMapper(Integer.class);
-        assertEquals(0, ((Integer) mappableMapper.convert(0, Object.class)).intValue());
-    }
-
-    @Test
-    public void testConvert2() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        assertEquals("source", mappableMapper.convert("source", Object.class));
-    }
-
-    @Test
-    public void testConvert3() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        assertEquals(0, ((Integer) mappableMapper.convert(0, Object.class)).intValue());
-    }
-
-    @Test
-    public void testConvertFromDocument() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        assertNull(mappableMapper.convertFromDocument(null, Object.class));
-    }
-
-    @Test
-    public void testConvertFromDocument2() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        Class type = Object.class;
-        assertThrows(ObjectMappingException.class,
-            () -> mappableMapper.convertFromDocument(Document.createDocument(), type));
-    }
-
-    @Test
-    public void testConvertFromDocument3() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        assertNull(
-            (new MappableMapper(forNameResult, forNameResult1, Object.class)).convertFromDocument(null, null));
-    }
-
-    @Test
-    public void testConvertToDocument() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        assertThrows(ObjectMappingException.class,
-            () -> (new MappableMapper(forNameResult, forNameResult1, Object.class)).convertToDocument("source"));
-    }
-
-    @Test
-    public void testConvertToDocument2() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        assertEquals(2, mappableMapper.convertToDocument(new NitriteTest.CompatChild()).size());
-    }
-
-    @Test
-    public void testConvertToDocument3() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        assertEquals(7, mappableMapper.convertToDocument(new NitriteStressTest.TestDto()).size());
-    }
-
-    @Test
-    public void testConvertToDocument4() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        assertEquals(3, mappableMapper.convertToDocument(new Company()).size());
-    }
-
-    @Test
-    public void testIsValueType() {
-        MappableMapper mappableMapper = new MappableMapper();
-        assertFalse(mappableMapper.isValueType(Object.class));
-    }
-
-    @Test
-    public void testIsValueType2() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper mappableMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
-        assertTrue(mappableMapper.isValueType(Object.class));
-    }
-
-    @Test
-    public void testIsValue() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        assertTrue((new MappableMapper(forNameResult, forNameResult1, Object.class)).isValue("object"));
-    }
-}
-
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MapperTest.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/MapperTest.java
index e8eb86a86..a7be25eda 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/mapper/MapperTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/mapper/MapperTest.java
@@ -31,14 +31,15 @@
  * @author Anindya Chatterjee
  */
 public class MapperTest {
-    private MappableMapper mappableMapper;
+    private SimpleDocumentMapper simpleDocumentMapper;
 
     @Rule
     public Retry retry = new Retry(3);
 
     @Test
     public void testWithConverter() {
-        mappableMapper = new MappableMapper();
+        simpleDocumentMapper = new SimpleDocumentMapper();
+        simpleDocumentMapper.registerEntityConverter(new Employee.Converter());
 
         Employee boss = new Employee();
         boss.setEmpId("1");
@@ -52,12 +53,12 @@ public void testWithConverter() {
         emp1.setBoss(boss);
 
         long start = System.currentTimeMillis();
-        Document document = mappableMapper.convert(emp1, Document.class);
+        Document document = simpleDocumentMapper.convert(emp1, Document.class);
         long diff = System.currentTimeMillis() - start;
         System.out.println(diff);
 
         start = System.currentTimeMillis();
-        Employee employee = mappableMapper.convert(document, Employee.class);
+        Employee employee = simpleDocumentMapper.convert(document, Employee.class);
         diff = System.currentTimeMillis() - start;
         System.out.println(diff);
         assertEquals(emp1, employee);
@@ -65,7 +66,10 @@ public void testWithConverter() {
 
     @Test
     public void testWithMappable() {
-        mappableMapper = new MappableMapper();
+        simpleDocumentMapper = new SimpleDocumentMapper();
+        simpleDocumentMapper.registerEntityConverter(new Department.DepartmentConverter());
+        simpleDocumentMapper.registerEntityConverter(new MappableEmployee.MappableEmployeeConverter());
+        simpleDocumentMapper.registerEntityConverter(new MappableDepartment.Converter());
 
         MappableEmployee boss = new MappableEmployee();
         boss.setEmpId("1");
@@ -79,12 +83,12 @@ public void testWithMappable() {
         emp1.setBoss(boss);
 
         long start = System.currentTimeMillis();
-        Document document = mappableMapper.convert(emp1, Document.class);
+        Document document = simpleDocumentMapper.convert(emp1, Document.class);
         long diff = System.currentTimeMillis() - start;
         System.out.println(diff);
 
         start = System.currentTimeMillis();
-        MappableEmployee employee = mappableMapper.convert(document, MappableEmployee.class);
+        MappableEmployee employee = simpleDocumentMapper.convert(document, MappableEmployee.class);
         diff = System.currentTimeMillis() - start;
         System.out.println(diff);
         assertEquals(emp1, employee);
@@ -92,7 +96,10 @@ public void testWithMappable() {
 
     @Test
     public void testWithConverterAndMappableMix() {
-        mappableMapper = new MappableMapper();
+        simpleDocumentMapper = new SimpleDocumentMapper();
+        simpleDocumentMapper.registerEntityConverter(new Department.DepartmentConverter());
+        simpleDocumentMapper.registerEntityConverter(new MappableEmployee.MappableEmployeeConverter());
+        simpleDocumentMapper.registerEntityConverter(new MappableDepartment.Converter());
 
         final MappableEmployee boss = new MappableEmployee();
         boss.setEmpId("1");
@@ -113,12 +120,12 @@ public void testWithConverterAndMappableMix() {
         }});
 
         long start = System.currentTimeMillis();
-        Document document = mappableMapper.convert(department, Document.class);
+        Document document = simpleDocumentMapper.convert(department, Document.class);
         long diff = System.currentTimeMillis() - start;
         System.out.println(diff);
 
         start = System.currentTimeMillis();
-        Department dept = mappableMapper.convert(document, Department.class);
+        Department dept = simpleDocumentMapper.convert(document, Department.class);
         diff = System.currentTimeMillis() - start;
         System.out.println(diff);
         assertEquals(department, dept);
@@ -126,7 +133,10 @@ public void testWithConverterAndMappableMix() {
 
     @Test
     public void testNested() {
-        mappableMapper = new MappableMapper();
+        simpleDocumentMapper = new SimpleDocumentMapper();
+        simpleDocumentMapper.registerEntityConverter(new Department.DepartmentConverter());
+        simpleDocumentMapper.registerEntityConverter(new MappableEmployee.MappableEmployeeConverter());
+        simpleDocumentMapper.registerEntityConverter(new MappableDepartment.Converter());
 
         final MappableEmployee boss = new MappableEmployee();
         boss.setEmpId("1");
@@ -147,12 +157,12 @@ public void testNested() {
         }});
 
         long start = System.currentTimeMillis();
-        Document document = mappableMapper.convert(department, Document.class);
+        Document document = simpleDocumentMapper.convert(department, Document.class);
         long diff = System.currentTimeMillis() - start;
         System.out.println(diff);
 
         start = System.currentTimeMillis();
-        MappableDepartment dept = mappableMapper.convert(document, MappableDepartment.class);
+        MappableDepartment dept = simpleDocumentMapper.convert(document, MappableDepartment.class);
         diff = System.currentTimeMillis() - start;
         System.out.println(diff);
         assertEquals(department, dept);
@@ -160,13 +170,16 @@ public void testNested() {
 
     @Test
     public void testWithValueType() {
-        mappableMapper = new MappableMapper();
+        simpleDocumentMapper = new SimpleDocumentMapper();
+        simpleDocumentMapper.registerEntityConverter(new Company.CompanyConverter());
+        simpleDocumentMapper.registerEntityConverter(new Company.CompanyId.CompanyIdConverter());
+
         Company company = new Company();
         company.setName("test");
         company.setId(1L);
         company.setCompanyId(new Company.CompanyId(1L));
 
-        Document document = mappableMapper.convert(company, Document.class);
+        Document document = simpleDocumentMapper.convert(company, Document.class);
         Object companyId = document.get("companyId");
         assertTrue(companyId instanceof Document);
     }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/mapper/SimpleDocumentMapperTest.java b/nitrite/src/test/java/org/dizitart/no2/common/mapper/SimpleDocumentMapperTest.java
new file mode 100644
index 000000000..f9aaebf42
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/common/mapper/SimpleDocumentMapperTest.java
@@ -0,0 +1,96 @@
+package org.dizitart.no2.common.mapper;
+
+import org.dizitart.no2.integration.NitriteStressTest;
+import org.dizitart.no2.integration.NitriteTest;
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.exceptions.ObjectMappingException;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class SimpleDocumentMapperTest {
+    @Test
+    public void testConvert() {
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(Integer.class);
+        assertEquals(0, ((Integer) simpleDocumentMapper.convert(0, Object.class)).intValue());
+    }
+
+    @Test
+    public void testConvert2() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
+        assertEquals("source", simpleDocumentMapper.convert("source", Object.class));
+    }
+
+    @Test
+    public void testConvert3() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
+        assertEquals(0, ((Integer) simpleDocumentMapper.convert(0, Object.class)).intValue());
+    }
+
+    @Test
+    public void testConvertFromDocument() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
+        assertNull(simpleDocumentMapper.convertFromDocument(null, Object.class));
+    }
+
+    @Test
+    public void testConvertFromDocument2() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
+        Class type = Object.class;
+        assertThrows(ObjectMappingException.class,
+            () -> simpleDocumentMapper.convertFromDocument(Document.createDocument(), type));
+    }
+
+    @Test
+    public void testConvertFromDocument3() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        assertNull(
+            (new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class)).convertFromDocument(null, null));
+    }
+
+    @Test
+    public void testConvertToDocument() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        assertThrows(ObjectMappingException.class,
+            () -> (new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class)).convertToDocument("source"));
+    }
+
+    @Test
+    public void testConvertToDocument2() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
+        simpleDocumentMapper.registerEntityConverter(new NitriteTest.CompatChild.CompatChildConverter());
+        assertEquals(2, simpleDocumentMapper.convertToDocument(new NitriteTest.CompatChild()).size());
+    }
+
+    @Test
+    public void testConvertToDocument3() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
+        simpleDocumentMapper.registerEntityConverter(new NitriteStressTest.TestDto.Converter());
+        assertEquals(7, simpleDocumentMapper.convertToDocument(new NitriteStressTest.TestDto()).size());
+    }
+
+    @Test
+    public void testConvertToDocument4() {
+        Class forNameResult = Object.class;
+        Class forNameResult1 = Object.class;
+        SimpleDocumentMapper simpleDocumentMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
+        simpleDocumentMapper.registerEntityConverter(new Company.CompanyConverter());
+        assertEquals(3, simpleDocumentMapper.convertToDocument(new Company()).size());
+    }
+
+}
+
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/meta/AttributesAwareTest.java b/nitrite/src/test/java/org/dizitart/no2/common/meta/AttributesAwareTest.java
new file mode 100644
index 000000000..888fa38e0
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/common/meta/AttributesAwareTest.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.common.meta;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNull;
+
+public class AttributesAwareTest {
+    @Test
+    public void testGetAttributes() {
+        AttributesAware attributesAware = new AttributesAware() {
+            @Override
+            public Attributes getAttributes() {
+                return new Attributes();
+            }
+
+            @Override
+            public void setAttributes(Attributes attributes) {
+
+            }
+        };
+        Assert.assertNotNull("This is a boilerplate assert on the result.", attributesAware.getAttributes());
+    }
+
+    @Test
+    public void testSetAttributes() {
+        AttributesAware attributesAware = new AttributesAware() {
+            @Override
+            public Attributes getAttributes() {
+                return null;
+            }
+
+            @Override
+            public void setAttributes(Attributes attributes) {
+                assertNull(attributes);
+            }
+        };
+        attributesAware.setAttributes(null);
+    }
+}
+
diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/meta/AttributesTest.java b/nitrite/src/test/java/org/dizitart/no2/common/meta/AttributesTest.java
similarity index 60%
rename from nitrite/src/test/java/org/dizitart/no2/collection/meta/AttributesTest.java
rename to nitrite/src/test/java/org/dizitart/no2/common/meta/AttributesTest.java
index 4689d2aeb..2f1fa9fe5 100644
--- a/nitrite/src/test/java/org/dizitart/no2/collection/meta/AttributesTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/meta/AttributesTest.java
@@ -1,4 +1,21 @@
-package org.dizitart.no2.collection.meta;
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.common.meta;
 
 import org.junit.Test;
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/module/PluginManagerTest.java b/nitrite/src/test/java/org/dizitart/no2/common/module/PluginManagerTest.java
index ceb44e2d5..459dd6a84 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/module/PluginManagerTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/module/PluginManagerTest.java
@@ -18,6 +18,7 @@
 package org.dizitart.no2.common.module;
 
 import org.dizitart.no2.NitriteConfig;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.exceptions.PluginException;
 import org.dizitart.no2.store.NitriteStore;
 import org.junit.Test;
@@ -62,7 +63,7 @@ public void testFindAndLoadPlugins() {
         NitriteStore nitriteStore = pluginManager.getNitriteStore();
         assertTrue(nitriteStore instanceof org.dizitart.no2.store.memory.InMemoryStore);
         assertEquals(3, pluginManager.getIndexerMap().size());
-        assertTrue(pluginManager.getNitriteMapper() instanceof org.dizitart.no2.common.mapper.MappableMapper);
+        assertTrue(pluginManager.getNitriteMapper() instanceof SimpleDocumentMapper);
         assertFalse(nitriteStore.isClosed());
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/streams/BoundedDocumentStreamTest.java b/nitrite/src/test/java/org/dizitart/no2/common/streams/BoundedStreamTest.java
similarity index 80%
rename from nitrite/src/test/java/org/dizitart/no2/common/streams/BoundedDocumentStreamTest.java
rename to nitrite/src/test/java/org/dizitart/no2/common/streams/BoundedStreamTest.java
index 4aa3a7054..6968e25a5 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/streams/BoundedDocumentStreamTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/streams/BoundedStreamTest.java
@@ -30,22 +30,22 @@
 import static org.junit.Assert.assertThrows;
 import static org.mockito.Mockito.*;
 
-public class BoundedDocumentStreamTest {
+public class BoundedStreamTest {
     @Test
     public void testConstructor() {
-        new BoundedDocumentStream(1L, 1L, (RecordStream>) mock(RecordStream.class));
+        new BoundedStream<>(1L, 1L, (RecordStream>) mock(RecordStream.class));
     }
 
     @Test
     public void testConstructor2() {
         assertThrows(ValidationException.class,
-                () -> new BoundedDocumentStream(-1L, 1L, (RecordStream>) mock(RecordStream.class)));
+                () -> new BoundedStream<>(-1L, 1L, (RecordStream>) mock(RecordStream.class)));
     }
 
     @Test
     public void testConstructor3() {
         assertThrows(ValidationException.class,
-                () -> new BoundedDocumentStream(1L, -1L, (RecordStream>) mock(RecordStream.class)));
+                () -> new BoundedStream<>(1L, -1L, (RecordStream>) mock(RecordStream.class)));
     }
 
     @Test
@@ -65,7 +65,8 @@ public Pair next() {
             }
         };
 
-        BoundedDocumentStream stream = new BoundedDocumentStream(2L, 1L, recordStream);
+        BoundedStream stream
+            = new BoundedStream<>(2L, 1L, recordStream);
 
         for (Pair pair : stream) {
             assertEquals(3, (int) pair.getSecond().get("value", Integer.class));
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/streams/UnionStreamTest.java b/nitrite/src/test/java/org/dizitart/no2/common/streams/ConcatStreamTest.java
similarity index 76%
rename from nitrite/src/test/java/org/dizitart/no2/common/streams/UnionStreamTest.java
rename to nitrite/src/test/java/org/dizitart/no2/common/streams/ConcatStreamTest.java
index 4d6724e39..49f870e30 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/streams/UnionStreamTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/streams/ConcatStreamTest.java
@@ -23,17 +23,17 @@
 
 import static org.junit.Assert.assertTrue;
 
-public class UnionStreamTest {
+public class ConcatStreamTest {
     @Test
     public void testConstructor() {
-        assertTrue((new UnionStream(new ArrayList<>())).toList().isEmpty());
+        assertTrue((new ConcatStream(new ArrayList<>())).toList().isEmpty());
     }
 
     @Test
     public void testIterator() {
-        UnionStream unionStream = new UnionStream(new ArrayList<>());
-        unionStream.iterator();
-        assertTrue(unionStream.toList().isEmpty());
+        ConcatStream concatStream = new ConcatStream(new ArrayList<>());
+        concatStream.iterator();
+        assertTrue(concatStream.toList().isEmpty());
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/streams/ProjectedDocumentStreamTest.java b/nitrite/src/test/java/org/dizitart/no2/common/streams/ProjectedDocumentStreamTest.java
index b07e1c288..1e76c1c74 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/streams/ProjectedDocumentStreamTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/streams/ProjectedDocumentStreamTest.java
@@ -18,11 +18,12 @@
 package org.dizitart.no2.common.streams;
 
 import com.fasterxml.jackson.databind.util.ArrayIterator;
+import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.RecordStream;
-import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.common.processors.ProcessorChain;
+import org.dizitart.no2.common.tuples.Pair;
 import org.junit.Test;
 
 import static org.junit.Assert.assertNull;
@@ -65,5 +66,25 @@ public void testIterator2() {
         verify(recordStream).iterator();
         assertTrue(projectedDocumentStream.toList().isEmpty());
     }
+    @Test
+    public void test() {
+        try(Nitrite db = Nitrite.builder().openOrCreate()) {
+            Document document = Document.createDocument("name", "John")
+                .put("address", Document.createDocument("street", "Main Street")
+                    .put("city", "New York")
+                    .put("state", "NY")
+                    .put("zip", "10001"));
+            db.getCollection("users").insert(document);
+
+            Document projection = Document.createDocument("name", null)
+                .put("address.city", null)
+                .put("address.state", null);
+
+            db.getCollection("users")
+                .find()
+                .project(projection)
+                .forEach(System.out::println);
+        }
+    }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/streams/SortedDocumentStreamTest.java b/nitrite/src/test/java/org/dizitart/no2/common/streams/SortedDocumentStreamTest.java
index 7156cd267..0dbc085ae 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/streams/SortedDocumentStreamTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/streams/SortedDocumentStreamTest.java
@@ -40,7 +40,8 @@ public void testConstructor() {
         List> blockingSortOrder = findPlan.getBlockingSortOrder();
         assertTrue(blockingSortOrder instanceof java.util.ArrayList);
         assertEquals("FindPlan(byIdFilter=null, indexScanFilter=null, collectionScanFilter=null, indexDescriptor=null,"
-            + " indexScanOrder=null, blockingSortOrder=[], skip=null, limit=null, collator=null, subPlans=[])", findPlan.toString());
+            + " indexScanOrder=null, blockingSortOrder=[], skip=null, limit=null, distinct=false, collator=null, " +
+            "subPlans=[])", findPlan.toString());
         assertTrue(blockingSortOrder.isEmpty());
         List subPlans = findPlan.getSubPlans();
         assertTrue(subPlans instanceof java.util.ArrayList);
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/util/ComparablesTest.java b/nitrite/src/test/java/org/dizitart/no2/common/util/ComparablesTest.java
index 7f853ad39..77ce794c4 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/util/ComparablesTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/util/ComparablesTest.java
@@ -29,7 +29,7 @@ public void testCompare3() {
     @Test
     public void testCompare4() {
         MutableByte first = new MutableByte();
-        assertEquals(1, Comparables.compare(first, new MutableDouble()));
+        assertEquals(-1, Comparables.compare(first, new MutableDouble()));
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/util/DocumentUtilsTest.java b/nitrite/src/test/java/org/dizitart/no2/common/util/DocumentUtilsTest.java
index 8776519f7..7cca9f4f5 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/util/DocumentUtilsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/util/DocumentUtilsTest.java
@@ -18,9 +18,9 @@
 
 import org.dizitart.no2.NitriteBuilderTest;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
-import org.dizitart.no2.common.mapper.MappableMapper;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.filters.ComparableFilter;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.integration.Retry;
@@ -83,10 +83,17 @@ public void testSkeletonDocument() {
     public void testSkeletonDocument2() {
         Class forNameResult = Object.class;
         Class forNameResult1 = Object.class;
-        MappableMapper nitriteMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
+        SimpleDocumentMapper nitriteMapper = new SimpleDocumentMapper(forNameResult, forNameResult1, Object.class);
         assertEquals(0, DocumentUtils.skeletonDocument(nitriteMapper, Object.class).size());
     }
 
+    @Test
+    public void testSkeletonDocument3() {
+        SimpleDocumentMapper nitriteMapper = new SimpleDocumentMapper();
+        Document document = DocumentUtils.skeletonDocument(nitriteMapper, Integer.class);
+        assertNull(document);
+    }
+
     @Test
     public void testIsSimilar() {
         assertFalse(DocumentUtils.isSimilar(null, Document.createDocument(), "fields"));
@@ -103,7 +110,9 @@ public void testIsSimilar2() {
 
     @Test
     public void testDummyDocument() {
-        NitriteMapper nitriteMapper = new MappableMapper();
+        SimpleDocumentMapper nitriteMapper = new SimpleDocumentMapper();
+        nitriteMapper.registerEntityConverter(new DummyTest.Converter());
+
         Document document = skeletonDocument(nitriteMapper, DummyTest.class);
         assertTrue(document.containsKey("first"));
         assertTrue(document.containsKey("second"));
@@ -111,20 +120,30 @@ public void testDummyDocument() {
         assertNull(document.get("second"));
     }
 
-    private static class DummyTest implements Mappable {
+    private static class DummyTest {
         private String first;
         private Double second;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return createDocument("first", first)
-                .put("second", second);
-        }
-
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            first = document.get("first", String.class);
-            second = document.get("second", Double.class);
+        public static class Converter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return DummyTest.class;
+            }
+
+            @Override
+            public Document toDocument(DummyTest entity, NitriteMapper nitriteMapper) {
+                return createDocument("first", entity.first)
+                    .put("second", entity.second);
+            }
+
+            @Override
+            public DummyTest fromDocument(Document document, NitriteMapper nitriteMapper) {
+                DummyTest entity = new DummyTest();
+                entity.first = document.get("first", String.class);
+                entity.second = document.get("second", Double.class);
+                return entity;
+            }
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/util/IndexUtilsTest.java b/nitrite/src/test/java/org/dizitart/no2/common/util/IndexUtilsTest.java
index d3de97e68..b25ba2933 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/util/IndexUtilsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/util/IndexUtilsTest.java
@@ -33,7 +33,7 @@ public void testDeriveIndexMapName() {
     @Test
     public void testDeriveIndexMapName2() {
         IndexDescriptor indexDescriptor = new IndexDescriptor("Index Type", new Fields(), "Collection Name");
-        indexDescriptor.setIndexFields(new Fields());
+        indexDescriptor.setFields(new Fields());
         assertEquals("$nitrite_index|Collection Name||Index Type", IndexUtils.deriveIndexMapName(indexDescriptor));
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/util/NumbersTest.java b/nitrite/src/test/java/org/dizitart/no2/common/util/NumbersTest.java
index 872ab873e..853a19943 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/util/NumbersTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/util/NumbersTest.java
@@ -78,7 +78,7 @@ public void testCompare() {
 
     @Test
     public void testCompare2() {
-        Integer x = new Integer(1);
-        assertEquals(0, Numbers.compare(x, new Integer(1)));
+        Integer x = 1;
+        assertEquals(0, Numbers.compare(x, 1));
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/util/ObjectUtilsTest.java b/nitrite/src/test/java/org/dizitart/no2/common/util/ObjectUtilsTest.java
index 8a6fea0db..c82b52df1 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/util/ObjectUtilsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/util/ObjectUtilsTest.java
@@ -21,19 +21,35 @@
 import lombok.Data;
 import org.apache.commons.lang3.mutable.MutableByte;
 import org.apache.commons.lang3.mutable.MutableDouble;
-import org.apache.lucene.analysis.CharArraySet;
-import org.apache.lucene.analysis.StopFilter;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
+import org.dizitart.no2.exceptions.ObjectMappingException;
 import org.dizitart.no2.exceptions.ValidationException;
-import org.dizitart.no2.repository.annotations.Entity;
-import org.dizitart.no2.repository.annotations.Index;
+import org.dizitart.no2.integration.NitriteTest;
 import org.dizitart.no2.integration.repository.data.ChildClass;
+import org.dizitart.no2.integration.repository.data.Company;
 import org.dizitart.no2.integration.repository.data.Employee;
+import org.dizitart.no2.integration.repository.data.Note;
+import org.dizitart.no2.integration.repository.decorator.Manufacturer;
+import org.dizitart.no2.integration.repository.decorator.ManufacturerDecorator;
+import org.dizitart.no2.integration.repository.decorator.ProductDecorator;
+import org.dizitart.no2.repository.annotations.Entity;
+import org.dizitart.no2.repository.annotations.Index;
 import org.junit.Test;
 
 import java.io.Serializable;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.URI;
+import java.net.URL;
+import java.time.Instant;
 import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.*;
+import java.util.regex.Pattern;
 
 import static org.dizitart.no2.common.util.ObjectUtils.newInstance;
 import static org.junit.Assert.*;
@@ -56,6 +72,16 @@ public void testFindRepositoryName() {
         assertEquals("java.lang.Object", ObjectUtils.findRepositoryName(Object.class, ""));
     }
 
+    @Test
+    public void testFindRepositoryNameByDecorator() {
+        assertEquals("product", ObjectUtils.findRepositoryNameByDecorator(new ProductDecorator(), ""));
+        assertEquals("product+key", ObjectUtils.findRepositoryNameByDecorator(new ProductDecorator(), "key"));
+        assertEquals(Manufacturer.class.getName() + "+key",
+            ObjectUtils.findRepositoryNameByDecorator(new ManufacturerDecorator(), "key"));
+        assertEquals(Manufacturer.class.getName(),
+            ObjectUtils.findRepositoryNameByDecorator(new ManufacturerDecorator(), ""));
+    }
+
     @Test
     public void testDeepEquals() {
         assertFalse(ObjectUtils.deepEquals("o1", "o2"));
@@ -67,13 +93,6 @@ public void testDeepEquals() {
         assertFalse(ObjectUtils.deepEquals("o1", null));
     }
 
-    @Test
-    public void testDeepEquals2() {
-        CharArraySet makeStopSetResult = StopFilter.makeStopSet(new String[]{"foo", "foo", "foo"}, true);
-        makeStopSetResult.add((Object) "foo");
-        assertFalse(ObjectUtils.deepEquals(makeStopSetResult, new AnnotatedMethodMap()));
-    }
-
     @Test
     public void testDeepEquals3() {
         MutableByte o1 = new MutableByte();
@@ -92,15 +111,6 @@ public void testDeepEquals5() {
         assertTrue(ObjectUtils.deepEquals(o1, new MutableByte()));
     }
 
-    @Test
-    public void testDeepEquals6() {
-        CharArraySet makeStopSetResult = StopFilter.makeStopSet(new String[]{"foo", "foo", "foo"}, true);
-        makeStopSetResult.add((Object) "foo");
-        CharArraySet makeStopSetResult1 = StopFilter.makeStopSet(new String[]{"foo", "foo", "foo"}, true);
-        makeStopSetResult1.add((Object) "foo");
-        assertTrue(ObjectUtils.deepEquals(makeStopSetResult, makeStopSetResult1));
-    }
-
     @Test
     public void testDeepEquals7() {
         AnnotatedMethodMap o1 = new AnnotatedMethodMap();
@@ -115,13 +125,46 @@ public void testDeepEquals8() {
 
     @Test
     public void testNewInstance() {
-        EnclosingType type = newInstance(EnclosingType.class, true);
-        System.out.println(type);
-    }
+        SimpleDocumentMapper mapper = new SimpleDocumentMapper();
+        mapper.registerEntityConverter(new EnclosingType.Converter());
+        mapper.registerEntityConverter(new ChildClass.Converter());
+        mapper.registerEntityConverter(new FieldType.Converter());
+        mapper.registerEntityConverter(new Employee.EmployeeConverter());
+        mapper.registerEntityConverter(new Company.CompanyConverter());
+        mapper.registerEntityConverter(new Note.NoteConverter());
 
-    @Test
-    public void testIsValueType() {
-        assertFalse(ObjectUtils.isValueType(Object.class));
+        EnclosingType type = newInstance(EnclosingType.class, true, mapper);
+        assertNotNull(type);
+
+        assertThrows(ObjectMappingException.class, () -> newInstance(Object.class, false, mapper));
+        assertThrows(ObjectMappingException.class, () -> newInstance(NitriteTest.Receipt.class, false, mapper));
+
+        assertNull(newInstance(byte[].class, false, mapper));
+        assertNull(newInstance(Number.class, false, mapper));
+        assertNull(newInstance(Byte.class, false, mapper));
+        assertNull(newInstance(Short.class, false, mapper));
+        assertNull(newInstance(Integer.class, false, mapper));
+        assertNull(newInstance(Long.class, false, mapper));
+        assertNull(newInstance(Float.class, false, mapper));
+        assertNull(newInstance(Double.class, false, mapper));
+        assertNull(newInstance(BigDecimal.class, false, mapper));
+        assertNull(newInstance(BigInteger.class, false, mapper));
+        assertNull(newInstance(Boolean.class, false, mapper));
+        assertNull(newInstance(Character.class, false, mapper));
+        assertNull(newInstance(String.class, false, mapper));
+        assertNull(newInstance(Date.class, false, mapper));
+        assertNull(newInstance(URL.class, false, mapper));
+        assertNull(newInstance(URI.class, false, mapper));
+        assertNull(newInstance(Currency.class, false, mapper));
+        assertNull(newInstance(Calendar.class, false, mapper));
+        assertNull(newInstance(StringBuffer.class, false, mapper));
+        assertNull(newInstance(StringBuilder.class, false, mapper));
+        assertNull(newInstance(Locale.class, false, mapper));
+        assertNull(newInstance(Void.class, false, mapper));
+        assertNull(newInstance(UUID.class, false, mapper));
+        assertNull(newInstance(Pattern.class, false, mapper));
+
+        assertNull(newInstance(GregorianCalendar.class, false, mapper));
     }
 
     @Test
@@ -167,12 +210,64 @@ public void testValidEntity() {
     private static class EnclosingType {
         private ChildClass childClass;
         private FieldType fieldType;
+
+        public static class Converter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return EnclosingType.class;
+            }
+
+            @Override
+            public Document toDocument(EnclosingType entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument()
+                    .put("childClass", nitriteMapper.convert(entity.childClass, Document.class))
+                    .put("fieldType", nitriteMapper.convert(entity.fieldType, Document.class));
+            }
+
+            @Override
+            public EnclosingType fromDocument(Document document, NitriteMapper nitriteMapper) {
+                EnclosingType entity = new EnclosingType();
+                entity.childClass = nitriteMapper.convert(document.get("childClass", Document.class),
+                    ChildClass.class);
+                entity.fieldType = nitriteMapper.convert(document.get("fieldType", Document.class),
+                    FieldType.class);
+                return entity;
+            }
+        }
     }
 
     @Data
     private static class FieldType {
         private Employee employee;
         private LocalDateTime currentDate;
+
+        public static class Converter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return FieldType.class;
+            }
+
+            @Override
+            public Document toDocument(FieldType entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument()
+                    .put("currentDate", entity.currentDate.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli())
+                    .put("employee", nitriteMapper.convert(entity.employee, Document.class));
+            }
+
+            @Override
+            public FieldType fromDocument(Document document, NitriteMapper nitriteMapper) {
+                FieldType entity = new FieldType();
+                entity.employee = nitriteMapper.convert(document.get("employee", Document.class), Employee.class);
+                if (document.get("currentDate", Long.class) != null) {
+                    entity.currentDate = LocalDateTime.ofInstant(
+                        Instant.ofEpochMilli(document.get("currentDate", Long.class)),
+                        ZoneId.systemDefault());
+                }
+                return entity;
+            }
+        }
     }
 
     @Data
@@ -201,7 +296,7 @@ private static class ValidEntity4 {
 
     @Data
     @Entity(indices = {
-        @Index(value = "value")
+        @Index(fields = "value")
     })
     private static class ValidEntity5 {
         private String value;
diff --git a/nitrite/src/test/java/org/dizitart/no2/common/util/ValidationUtilsTest.java b/nitrite/src/test/java/org/dizitart/no2/common/util/ValidationUtilsTest.java
index cd7721924..10c373a75 100644
--- a/nitrite/src/test/java/org/dizitart/no2/common/util/ValidationUtilsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/common/util/ValidationUtilsTest.java
@@ -16,15 +16,18 @@
 
 package org.dizitart.no2.common.util;
 
-import org.dizitart.no2.integration.Retry;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.exceptions.ValidationException;
+import org.dizitart.no2.integration.Retry;
+import org.dizitart.no2.integration.repository.data.ClassA;
+import org.dizitart.no2.integration.repository.data.ClassBConverter;
+import org.dizitart.no2.integration.repository.data.ClassC;
+import org.dizitart.no2.integration.repository.data.EmptyClass;
 import org.junit.Rule;
 import org.junit.Test;
 
-import static org.dizitart.no2.common.util.ValidationUtils.notEmpty;
-import static org.dizitart.no2.common.util.ValidationUtils.notNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.dizitart.no2.common.util.ValidationUtils.*;
+import static org.junit.Assert.*;
 
 /**
  * @author Anindya Chatterjee.
@@ -73,4 +76,38 @@ public void testCharSequenceNotEmpty() {
             assertTrue(invalid);
         }
     }
+
+    @Test
+    public void testValidateProjectionType() {
+        SimpleDocumentMapper documentMapper = new SimpleDocumentMapper();
+        documentMapper.registerEntityConverter(new ClassA.ClassAConverter());
+        documentMapper.registerEntityConverter(new ClassBConverter());
+        documentMapper.registerEntityConverter(new EmptyClass.Converter());
+
+        validateProjectionType(ClassA.class, documentMapper);
+
+        assertThrows(ValidationException.class, () -> validateProjectionType(EmptyClass.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateProjectionType(ClassC.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateProjectionType(String.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateProjectionType(Number.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateProjectionType(Integer.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateProjectionType(Object.class, documentMapper));
+    }
+
+    @Test
+    public void testValidateRepositoryType() {
+        SimpleDocumentMapper documentMapper = new SimpleDocumentMapper();
+        documentMapper.registerEntityConverter(new ClassA.ClassAConverter());
+        documentMapper.registerEntityConverter(new ClassBConverter());
+        documentMapper.registerEntityConverter(new EmptyClass.Converter());
+
+        validateRepositoryType(ClassA.class, documentMapper);
+
+        assertThrows(ValidationException.class, () -> validateRepositoryType(EmptyClass.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateRepositoryType(ClassC.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateRepositoryType(String.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateRepositoryType(Number.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateRepositoryType(Integer.class, documentMapper));
+        assertThrows(ValidationException.class, () -> validateRepositoryType(Object.class, documentMapper));
+    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/filters/BetweenFilterTest.java b/nitrite/src/test/java/org/dizitart/no2/filters/BetweenFilterTest.java
index cc1aecc29..2f43255e0 100644
--- a/nitrite/src/test/java/org/dizitart/no2/filters/BetweenFilterTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/filters/BetweenFilterTest.java
@@ -17,7 +17,7 @@
 
 package org.dizitart.no2.filters;
 
-import org.dizitart.no2.exceptions.ValidationException;
+import org.dizitart.no2.exceptions.FilterException;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
@@ -84,19 +84,19 @@ public void testConstructor() {
 
     @Test
     public void testConstructor2() {
-        assertThrows(ValidationException.class,
+        assertThrows(FilterException.class,
                 () -> new BetweenFilter<>("Field", new BetweenFilter.Bound<>(null, "Upper Bound")));
     }
 
     @Test
     public void testConstructor3() {
-        assertThrows(ValidationException.class,
+        assertThrows(FilterException.class,
                 () -> new BetweenFilter<>("Field", new BetweenFilter.Bound<>("Lower Bound", null)));
     }
 
     @Test
     public void testConstructor4() {
-        assertThrows(ValidationException.class, () -> new BetweenFilter<>("Field", null));
+        assertThrows(FilterException.class, () -> new BetweenFilter<>("Field", null));
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/filters/TextFilterTest.java b/nitrite/src/test/java/org/dizitart/no2/filters/TextFilterTest.java
index c942d0e33..a9066378f 100644
--- a/nitrite/src/test/java/org/dizitart/no2/filters/TextFilterTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/filters/TextFilterTest.java
@@ -52,7 +52,7 @@ public void testToString2() {
     public void testApplyOnIndex() {
         TextFilter textFilter = new TextFilter("Field", "42");
         textFilter.setTextTokenizer(new EnglishTextTokenizer());
-        assertTrue(textFilter.applyOnIndex(new InMemoryMap<>("Map Name", null)).isEmpty());
+        assertTrue(textFilter.applyOnTextIndex(new InMemoryMap<>("Map Name", null)).isEmpty());
         assertEquals("42", textFilter.getStringValue());
     }
 
@@ -61,7 +61,7 @@ public void testApplyOnIndex3() {
         TextFilter textFilter = new TextFilter("Field", "*");
         textFilter.setTextTokenizer(new EnglishTextTokenizer());
         assertThrows(FilterException.class,
-                () -> textFilter.applyOnIndex(new InMemoryMap<>("Map Name", null)));
+                () -> textFilter.applyOnTextIndex(new InMemoryMap<>("Map Name", null)));
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/index/EntityIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/index/EntityIndexTest.java
new file mode 100644
index 000000000..f58c9f23b
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/index/EntityIndexTest.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.index;
+
+import org.dizitart.no2.exceptions.ValidationException;
+import org.dizitart.no2.repository.EntityIndex;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class EntityIndexTest {
+
+    @Test
+    public void testIndextype() {
+        EntityIndex entityIndex = new EntityIndex(IndexType.UNIQUE, "a", "b");
+        assertEquals(entityIndex.getIndexType(), IndexType.UNIQUE);
+    }
+
+    @Test
+    public void testGetFields() {
+        EntityIndex entityIndex = new EntityIndex(IndexType.UNIQUE, "a", "b");
+        assertArrayEquals(entityIndex.getFieldNames().toArray(), new String[] {"a", "b"});
+    }
+
+    @Test(expected = ValidationException.class)
+    public void testGetFieldsWithoutFields() {
+        EntityIndex entityIndex = new EntityIndex(IndexType.UNIQUE);
+        assertArrayEquals(entityIndex.getFieldNames().toArray(), new String[0]);
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/index/IndexMapTest.java b/nitrite/src/test/java/org/dizitart/no2/index/IndexMapTest.java
index 647b16da4..67ba01f71 100644
--- a/nitrite/src/test/java/org/dizitart/no2/index/IndexMapTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/index/IndexMapTest.java
@@ -18,6 +18,7 @@
 package org.dizitart.no2.index;
 
 import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.common.DBValue;
 import org.dizitart.no2.store.memory.InMemoryMap;
 import org.junit.Test;
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/index/NitriteTextIndexerTest.java b/nitrite/src/test/java/org/dizitart/no2/index/NitriteTextIndexerTest.java
index 4ccfe6ca9..daece55b9 100644
--- a/nitrite/src/test/java/org/dizitart/no2/index/NitriteTextIndexerTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/index/NitriteTextIndexerTest.java
@@ -77,11 +77,11 @@ public void testDropIndex2() {
         NitriteTextIndexer nitriteTextIndexer = new NitriteTextIndexer();
         IndexDescriptor indexDescriptor = mock(IndexDescriptor.class);
         when(indexDescriptor.getIndexType()).thenThrow(new IndexingException("An error occurred"));
-        when(indexDescriptor.getIndexFields()).thenReturn(new Fields());
+        when(indexDescriptor.getFields()).thenReturn(new Fields());
         when(indexDescriptor.getCollectionName()).thenReturn("foo");
         nitriteTextIndexer.dropIndex(indexDescriptor, new NitriteConfig());
         verify(indexDescriptor).getIndexType();
-        verify(indexDescriptor).getIndexFields();
+        verify(indexDescriptor).getFields();
         verify(indexDescriptor).getCollectionName();
     }
 
@@ -90,13 +90,13 @@ public void testDropIndex3() {
         NitriteTextIndexer nitriteTextIndexer = new NitriteTextIndexer();
         IndexDescriptor indexDescriptor = mock(IndexDescriptor.class);
         when(indexDescriptor.getIndexType()).thenReturn("foo");
-        when(indexDescriptor.getIndexFields()).thenReturn(new Fields());
+        when(indexDescriptor.getFields()).thenReturn(new Fields());
         when(indexDescriptor.getCollectionName()).thenReturn("foo");
         NitriteConfig nitriteConfig = mock(NitriteConfig.class);
         doReturn(new InMemoryStore()).when(nitriteConfig).getNitriteStore();
         nitriteTextIndexer.dropIndex(indexDescriptor, nitriteConfig);
         verify(indexDescriptor).getIndexType();
-        verify(indexDescriptor).getIndexFields();
+        verify(indexDescriptor).getFields();
         verify(indexDescriptor).getCollectionName();
         verify(nitriteConfig).getNitriteStore();
     }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java
index 50426568f..7c2c22836 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/NitriteStressTest.java
@@ -22,8 +22,9 @@
 import lombok.Data;
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.index.IndexOptions;
 import org.dizitart.no2.index.IndexType;
 import org.dizitart.no2.repository.ObjectRepository;
@@ -52,6 +53,8 @@ public class NitriteStressTest {
     @Test
     public void stressTest() {
         Nitrite database = createDb();
+        SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) database.getConfig().nitriteMapper();
+        documentMapper.registerEntityConverter(new TestDto.Converter());
         ObjectRepository testRepository = database.getRepository(TestDto.class);
         testRepository.createIndex(IndexOptions.indexOptions(IndexType.FULL_TEXT), "lastName");
         testRepository.createIndex(IndexOptions.indexOptions(IndexType.NON_UNIQUE), "birthDate");
@@ -78,7 +81,7 @@ private List createTestSet() {
     }
 
     @Data
-    public static class TestDto implements Mappable {
+    public static class TestDto {
 
         @XmlElement(
             name = "StudentNumber",
@@ -126,27 +129,37 @@ public static class TestDto implements Mappable {
         public TestDto() {
         }
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return Document.createDocument()
-                .put("studentNumber", studentNumber)
-                .put("lastName", lastName)
-                .put("prefixes", prefixes)
-                .put("initials", initials)
-                .put("firstNames", firstNames)
-                .put("nickName", nickName)
-                .put("birthDate", birthDate);
-        }
+        public static class Converter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return TestDto.class;
+            }
 
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            studentNumber = document.get("studentNumber", String.class);
-            lastName = document.get("lastName", String.class);
-            prefixes = document.get("prefixes", String.class);
-            initials = document.get("initials", String.class);
-            firstNames = document.get("firstNames", String.class);
-            nickName = document.get("nickName", String.class);
-            birthDate = document.get("birthDate", String.class);
+            @Override
+            public Document toDocument(TestDto entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument()
+                    .put("studentNumber", entity.studentNumber)
+                    .put("lastName", entity.lastName)
+                    .put("prefixes", entity.prefixes)
+                    .put("initials", entity.initials)
+                    .put("firstNames", entity.firstNames)
+                    .put("nickName", entity.nickName)
+                    .put("birthDate", entity.birthDate);
+            }
+
+            @Override
+            public TestDto fromDocument(Document document, NitriteMapper nitriteMapper) {
+                TestDto entity = new TestDto();
+                entity.studentNumber = document.get("studentNumber", String.class);
+                entity.lastName = document.get("lastName", String.class);
+                entity.prefixes = document.get("prefixes", String.class);
+                entity.initials = document.get("initials", String.class);
+                entity.firstNames = document.get("firstNames", String.class);
+                entity.nickName = document.get("nickName", String.class);
+                entity.birthDate = document.get("birthDate", String.class);
+                return entity;
+            }
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/NitriteTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/NitriteTest.java
index 76055bfd1..a67921789 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/NitriteTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/NitriteTest.java
@@ -28,12 +28,14 @@
 import org.dizitart.no2.common.SortOrder;
 import org.dizitart.no2.common.WriteResult;
 import org.dizitart.no2.common.concurrent.ThreadPoolManager;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.exceptions.NitriteIOException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.dizitart.no2.index.IndexOptions;
 import org.dizitart.no2.index.IndexType;
-import org.dizitart.no2.common.mapper.Mappable;
-import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.integration.repository.data.EmptyClass;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.annotations.Id;
 import org.dizitart.no2.repository.annotations.Index;
@@ -54,12 +56,12 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 
-import static org.dizitart.no2.integration.TestUtil.createDb;
 import static org.dizitart.no2.collection.Document.createDocument;
 import static org.dizitart.no2.common.Constants.INTERNAL_NAME_SEPARATOR;
 import static org.dizitart.no2.common.Constants.META_MAP_NAME;
 import static org.dizitart.no2.filters.Filter.ALL;
 import static org.dizitart.no2.filters.FluentFilter.where;
+import static org.dizitart.no2.integration.TestUtil.createDb;
 import static org.junit.Assert.*;
 
 /**
@@ -74,6 +76,10 @@ public class NitriteTest {
     @Before
     public void setUp() throws ParseException {
         db = createDb("test-user", "test-password");
+        SimpleDocumentMapper nitriteMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        nitriteMapper.registerEntityConverter(new Receipt.ReceiptConverter());
+        nitriteMapper.registerEntityConverter(new CompatChild.CompatChildConverter());
+        nitriteMapper.registerEntityConverter(new EmptyClass.Converter());
 
         SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH);
 
@@ -122,9 +128,14 @@ public void testListCollectionNames() {
         assertEquals(collectionNames.size(), 1);
     }
 
-    @Test
+    @Test(expected = ValidationException.class)
     public void testListRepositories() {
         db.getRepository(getClass());
+    }
+
+    @Test
+    public void testListRepositories2() {
+        db.getRepository(Receipt.class);
         Set repositories = db.listRepositories();
         assertEquals(repositories.size(), 1);
     }
@@ -135,10 +146,15 @@ public void testHasCollection() {
         assertFalse(db.hasCollection("lucene" + INTERNAL_NAME_SEPARATOR + "test"));
     }
 
-    @Test
+    @Test(expected = ValidationException.class)
     public void testHasRepository() {
         db.getRepository(getClass());
-        assertTrue(db.hasRepository(getClass()));
+    }
+
+    @Test
+    public void testHasRepository2() {
+        db.getRepository(Receipt.class);
+        assertTrue(db.hasRepository(Receipt.class));
         assertFalse(db.hasRepository(String.class));
     }
 
@@ -158,20 +174,30 @@ public void testGetCollection() {
         assertEquals(collection.getName(), "test-collection");
     }
 
-    @Test
+    @Test(expected = ValidationException.class)
     public void testGetRepository() {
-        ObjectRepository repository = db.getRepository(NitriteTest.class);
-        assertNotNull(repository);
-        assertEquals(repository.getType(), NitriteTest.class);
+        ObjectRepository repository = db.getRepository(EmptyClass.class);
     }
 
     @Test
+    public void testGetRepository2() {
+        ObjectRepository repository = db.getRepository(Receipt.class);
+        assertNotNull(repository);
+        assertEquals(repository.getType(), Receipt.class);
+    }
+
+    @Test(expected = ValidationException.class)
     public void testGetRepositoryWithKey() {
-        ObjectRepository repository = db.getRepository(NitriteTest.class, "key");
+        ObjectRepository repository = db.getRepository(EmptyClass.class, "key");
+    }
+
+    @Test
+    public void testGetRepositoryWithKey2() {
+        ObjectRepository repository = db.getRepository(Receipt.class, "key");
         assertNotNull(repository);
-        assertEquals(repository.getType(), NitriteTest.class);
-        assertFalse(db.hasRepository(NitriteTest.class));
-        assertTrue(db.hasRepository(NitriteTest.class, "key"));
+        assertEquals(repository.getType(), Receipt.class);
+        assertFalse(db.hasRepository(Receipt.class));
+        assertTrue(db.hasRepository(Receipt.class, "key"));
     }
 
     @Test
@@ -187,18 +213,18 @@ public void testMultipleGetCollection() {
 
     @Test
     public void testMultipleGetRepository() {
-        ObjectRepository repository = db.getRepository(NitriteTest.class);
+        ObjectRepository repository = db.getRepository(Receipt.class);
         assertNotNull(repository);
-        assertEquals(repository.getType(), NitriteTest.class);
+        assertEquals(repository.getType(), Receipt.class);
 
-        ObjectRepository repository2 = db.getRepository(NitriteTest.class);
+        ObjectRepository repository2 = db.getRepository(Receipt.class);
         assertNotNull(repository2);
-        assertEquals(repository2.getType(), NitriteTest.class);
+        assertEquals(repository2.getType(), Receipt.class);
     }
 
     @Test(expected = ValidationException.class)
     public void testGetRepositoryInvalid() {
-        db.getRepository(null);
+        db.getRepository((Class) null);
     }
 
     @Test(expected = NitriteIOException.class)
@@ -212,14 +238,14 @@ public void testGetCollectionNullStore() {
     public void testGetRepositoryNullStore() {
         db = Nitrite.builder().openOrCreate();
         db.close();
-        db.getRepository(NitriteTest.class);
+        db.getRepository(EmptyClass.class);
     }
 
     @Test(expected = NitriteIOException.class)
     public void testGetKeyedRepositoryNullStore() {
         db = Nitrite.builder().openOrCreate();
         db.close();
-        db.getRepository(NitriteTest.class, "key");
+        db.getRepository(EmptyClass.class, "key");
     }
 
     @Test(expected = NitriteIOException.class)
@@ -377,20 +403,30 @@ public void run() {
     @Data
     @AllArgsConstructor
     @NoArgsConstructor
-    public static class CompatChild implements Mappable {
+    public static class CompatChild {
         private Long childId;
         private String lastName;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return Document.createDocument("childId", childId)
-                .put("lastName", lastName);
-        }
+        public static class CompatChildConverter implements EntityConverter {
 
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            childId = document.get("childId", Long.class);
-            lastName = document.get("lastName", String.class);
+            @Override
+            public Class getEntityType() {
+                return CompatChild.class;
+            }
+
+            @Override
+            public Document toDocument(CompatChild entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument("childId", entity.getChildId())
+                    .put("lastName", entity.getLastName());
+            }
+
+            @Override
+            public CompatChild fromDocument(Document document, NitriteMapper nitriteMapper) {
+                CompatChild compatChild = new CompatChild();
+                compatChild.setChildId(document.get("childId", Long.class));
+                compatChild.setLastName(document.get("lastName", String.class));
+                return compatChild;
+            }
         }
     }
 
@@ -398,41 +434,54 @@ public void read(NitriteMapper mapper, Document document) {
     @NoArgsConstructor
     @AllArgsConstructor
     @Indices({
-        @Index(value = "synced", type = IndexType.NON_UNIQUE)
+        @Index(fields = "synced", type = IndexType.NON_UNIQUE)
     })
-    public static class Receipt implements Mappable {
+    public static class Receipt {
         @Id
         private String clientRef;
         private Boolean synced;
         private Long createdTimestamp = System.currentTimeMillis();
         private Status status;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return createDocument("status", status)
-                .put("clientRef", clientRef)
-                .put("synced", synced)
-                .put("createdTimestamp", createdTimestamp);
+        public enum Status {
+            COMPLETED,
+            PREPARING,
         }
 
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            if (document != null) {
-                Object status = document.get("status");
-                if (status instanceof Status) {
-                    this.status = (Status) status;
-                } else {
-                    this.status = Status.valueOf(status.toString());
-                }
-                this.clientRef = document.get("clientRef", String.class);
-                this.synced = document.get("synced", Boolean.class);
-                this.createdTimestamp = document.get("createdTimestamp", Long.class);
+        public static class ReceiptConverter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return Receipt.class;
             }
-        }
 
-        public enum Status {
-            COMPLETED,
-            PREPARING,
+            @Override
+            public Document toDocument(Receipt entity, NitriteMapper nitriteMapper) {
+                return createDocument("status", entity.getStatus())
+                    .put("clientRef", entity.getClientRef())
+                    .put("synced", entity.getSynced())
+                    .put("createdTimestamp", entity.getCreatedTimestamp());
+            }
+
+            @Override
+            public Receipt fromDocument(Document document, NitriteMapper nitriteMapper) {
+                Receipt receipt = new Receipt();
+
+                if (document != null) {
+                    Object status = document.get("status");
+                    if (status != null) {
+                        if (status instanceof Receipt.Status) {
+                            receipt.status = (Receipt.Status) status;
+                        } else {
+                            receipt.status = Receipt.Status.valueOf(status.toString());
+                        }
+                    }
+                    receipt.clientRef = document.get("clientRef", String.class);
+                    receipt.synced = document.get("synced", Boolean.class);
+                    receipt.createdTimestamp = document.get("createdTimestamp", Long.class);
+                }
+                return receipt;
+            }
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/StressTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/StressTest.java
index 65142470c..437b293c9 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/StressTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/StressTest.java
@@ -22,11 +22,12 @@
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.DocumentCursor;
 import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.index.IndexOptions;
 import org.dizitart.no2.index.IndexType;
-import org.dizitart.no2.common.mapper.Mappable;
-import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.annotations.Index;
 import org.dizitart.no2.repository.annotations.Indices;
@@ -61,6 +62,10 @@ public void before() {
             .fieldSeparator(".")
             .openOrCreate();
 
+        SimpleDocumentMapper mapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        mapper.registerEntityConverter(new PerfTest.PerfTestConverter());
+        mapper.registerEntityConverter(new PerfTestIndexed.PerfTestIndexedConverter());
+
         collection = db.getCollection("test");
     }
 
@@ -182,36 +187,73 @@ private  List getItems(Class type) {
     }
 
     @Data
-    public static class PerfTest implements Mappable {
+    public static class PerfTest {
         private String firstName;
         private String lastName;
         private Integer age;
         private String text;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            Document document = Document.createDocument();
-            document.put("firstName", firstName);
-            document.put("lastName", lastName);
-            document.put("age", age);
-            document.put("text", text);
-            return document;
-        }
+        public static class PerfTestConverter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return PerfTest.class;
+            }
+
+            @Override
+            public Document toDocument(PerfTest entity, NitriteMapper nitriteMapper) {
+                Document document = Document.createDocument();
+                document.put("firstName", entity.firstName);
+                document.put("lastName", entity.lastName);
+                document.put("age", entity.age);
+                document.put("text", entity.text);
+                return document;
+            }
 
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            this.firstName = (String) document.get("firstName");
-            this.lastName = (String) document.get("lastName");
-            this.age = (Integer) document.get("age");
-            this.text = (String) document.get("text");
+            @Override
+            public PerfTest fromDocument(Document document, NitriteMapper nitriteMapper) {
+                PerfTest entity = new PerfTest();
+                entity.firstName = (String) document.get("firstName");
+                entity.lastName = (String) document.get("lastName");
+                entity.age = (Integer) document.get("age");
+                entity.text = (String) document.get("text");
+                return entity;
+            }
         }
     }
 
     @Indices({
-        @Index(value = "firstName", type = IndexType.NON_UNIQUE),
-        @Index(value = "age", type = IndexType.NON_UNIQUE),
-        @Index(value = "text", type = IndexType.FULL_TEXT),
+        @Index(fields = "firstName", type = IndexType.NON_UNIQUE),
+        @Index(fields = "age", type = IndexType.NON_UNIQUE),
+        @Index(fields = "text", type = IndexType.FULL_TEXT),
     })
     private static class PerfTestIndexed extends PerfTest {
+        public static class PerfTestIndexedConverter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return PerfTestIndexed.class;
+            }
+
+            @Override
+            public Document toDocument(PerfTestIndexed entity, NitriteMapper nitriteMapper) {
+                Document document = Document.createDocument();
+                document.put("firstName", entity.getFirstName());
+                document.put("lastName", entity.getLastName());
+                document.put("age", entity.getAge());
+                document.put("text", entity.getText());
+                return document;
+            }
+
+            @Override
+            public PerfTestIndexed fromDocument(Document document, NitriteMapper nitriteMapper) {
+                PerfTestIndexed entity = new PerfTestIndexed();
+                entity.setFirstName((String) document.get("firstName"));
+                entity.setLastName((String) document.get("lastName"));
+                entity.setAge((Integer) document.get("age"));
+                entity.setText((String) document.get("text"));
+                return entity;
+            }
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/TestUtil.java b/nitrite/src/test/java/org/dizitart/no2/integration/TestUtil.java
index c1d9e4b89..39e5a39d1 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/TestUtil.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/TestUtil.java
@@ -85,7 +85,7 @@ public static Document parse(String json) {
             return loadDocument(node);
         } catch (IOException e) {
             log.error("Error while parsing json", e);
-            throw new ObjectMappingException("failed to parse json " + json);
+            throw new ObjectMappingException("Failed to parse json " + json);
         }
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java
index 9dee23797..3278172eb 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/BaseCollectionTest.java
@@ -78,7 +78,7 @@ public void setUp() {
                     .put("lastName", "ln2")
                     .put("birthDay", simpleDateFormat.parse("2010-06-12T16:02:48.440Z"))
                     .put("data", new byte[]{3, 4, 3})
-                    .put("list", Arrays.asList("three", "four", "three"))
+                    .put("list", Arrays.asList("three", "four", "five"))
                     .put("body", "quick hello world from nitrite");
             doc3 = createDocument("firstName", "fn3")
                     .put("lastName", "ln2")
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionCompoundIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionCompoundIndexTest.java
index e0ce0646a..e2a4e4279 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionCompoundIndexTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionCompoundIndexTest.java
@@ -177,9 +177,9 @@ public void testNullValuesInIndexedFields() {
         insert();
         collection.insert(document);
 
-        DocumentCursor cursor = collection.find(where("fistName").eq(null));
+        DocumentCursor cursor = collection.find(where("firstName").eq(null));
         assertEquals("ln1", cursor.firstOrNull().get("lastName", String.class));
-        assertNull(cursor.firstOrNull().get("fistName", String.class));
+        assertNull(cursor.firstOrNull().get("firstName", String.class));
 
         document = createDocument("firstName", "fn4")
             .put("lastName", null)
@@ -195,7 +195,7 @@ public void testNullValuesInIndexedFields() {
 
         cursor = collection.find(where("lastName").eq(null));
         assertEquals("fn4", cursor.firstOrNull().get("firstName", String.class));
-        assertNull(cursor.firstOrNull().get("fistName", String.class));
+        assertNull(cursor.firstOrNull().get("lastName", String.class));
 
         cursor = collection.find(and(where("lastName").eq(null), where("birthDay").eq(null)));
         assertNull(cursor.firstOrNull().get("lastName", String.class));
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionDeleteTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionDeleteTest.java
index ca1660599..541621ced 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionDeleteTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionDeleteTest.java
@@ -90,7 +90,7 @@ public void testClear() {
             assertTrue(uniqueError);
         }
 
-        collection.remove(Filter.ALL);
+        collection.clear();
 
         cursor = collection.find();
         assertEquals(cursor.size(), 0);
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java
index 8fd30693e..3379c1dca 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindByCompoundIndexTest.java
@@ -19,6 +19,7 @@
 
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.DocumentCursor;
+import org.dizitart.no2.collection.FindOptions;
 import org.dizitart.no2.collection.FindPlan;
 import org.dizitart.no2.common.SortOrder;
 import org.dizitart.no2.common.tuples.Pair;
@@ -85,7 +86,7 @@ public void testFindByOrFilterAndFilter() {
             )
         );
 
-        assertEquals(2, cursor.size());
+        assertEquals(3, cursor.size());
 
         FindPlan findPlan = cursor.getFindPlan();
         assertNull(findPlan.getIndexScanFilter());
@@ -96,6 +97,39 @@ public void testFindByOrFilterAndFilter() {
         assertNotNull(findPlan.getSubPlans().get(0).getIndexScanFilter());
         assertNotNull(findPlan.getSubPlans().get(1).getIndexScanFilter());
 
+        assertEquals(1, cursor.toList().stream().filter(d ->
+            d.get("firstName", String.class).equals("fn2")
+                && d.get("lastName", String.class).equals("ln2")).count());
+
+        assertEquals(2, cursor.toList().stream().filter(d ->
+            d.get("firstName", String.class).equals("fn3")
+                && d.get("lastName", String.class).equals("ln2")).count());
+
+        // distinct test
+        cursor = collection.find(
+            or(
+                and(
+                    where("lastName").eq("ln2"),
+                    where("firstName").notEq("fn1")
+                ),
+                and(
+                    where("firstName").eq("fn3"),
+                    where("lastName").eq("ln2")
+                )
+            ), FindOptions.withDistinct()
+        );
+
+        assertEquals(2, cursor.size());
+
+        findPlan = cursor.getFindPlan();
+        assertNull(findPlan.getIndexScanFilter());
+        assertNull(findPlan.getCollectionScanFilter());
+        assertNotNull(findPlan.getSubPlans());
+
+        assertEquals(2, findPlan.getSubPlans().size());
+        assertNotNull(findPlan.getSubPlans().get(0).getIndexScanFilter());
+        assertNotNull(findPlan.getSubPlans().get(1).getIndexScanFilter());
+
         assertEquals(1, cursor.toList().stream().filter(d ->
             d.get("firstName", String.class).equals("fn2")
                 && d.get("lastName", String.class).equals("ln2")).count());
@@ -199,6 +233,22 @@ public void testFindByOrFilter() throws ParseException {
 
         FindPlan findPlan = cursor.getFindPlan();
         assertEquals(3, findPlan.getSubPlans().size());
+        assertEquals(5, cursor.size());
+
+        // distinct
+        cursor = collection.find(
+            or(
+                or(
+                    where("lastName").eq("ln2"),
+                    where("firstName").notEq("fn1")
+                ),
+                where("birthDay").eq(simpleDateFormat.parse("2012-07-01T16:02:48.440Z")),
+                where("firstName").notEq("fn1")
+            ), FindOptions.withDistinct()
+        );
+
+        findPlan = cursor.getFindPlan();
+        assertEquals(3, findPlan.getSubPlans().size());
         assertEquals(3, cursor.size());
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java
index c0e84a688..e1ecacaa5 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindNegativeTest.java
@@ -21,6 +21,7 @@
 import org.dizitart.no2.collection.DocumentCursor;
 import org.dizitart.no2.common.SortOrder;
 import org.dizitart.no2.exceptions.FilterException;
+import org.dizitart.no2.exceptions.InvalidOperationException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.junit.Test;
 
@@ -55,7 +56,7 @@ public void testFindOptionsNegativeSize() {
         collection.find(skipBy(0).limit(-1));
     }
 
-    @Test(expected = ValidationException.class)
+    @Test(expected = InvalidOperationException.class)
     public void testFindInvalidSort() {
         insert();
         collection.find(orderBy("data", SortOrder.Descending)).toList();
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java
index 25061010c..96c55ee3f 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindTest.java
@@ -21,7 +21,9 @@
 import org.dizitart.no2.collection.DocumentCursor;
 import org.dizitart.no2.collection.NitriteCollection;
 import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.common.RecordStream;
 import org.dizitart.no2.common.SortOrder;
+import org.dizitart.no2.common.processors.StringFieldEncryptionProcessor;
 import org.dizitart.no2.exceptions.IndexingException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.dizitart.no2.index.IndexOptions;
@@ -272,6 +274,12 @@ public void testFindSortOnNonExistingField() {
         insert();
         DocumentCursor cursor = collection.find(orderBy("my-value", SortOrder.Descending));
         assertEquals(cursor.size(), 3);
+
+        List dateList = new ArrayList<>();
+        for (Document document : cursor) {
+            dateList.add(document.get("birthDay", Date.class));
+        }
+        assertFalse(isSorted(dateList, true));
     }
 
     @Test
@@ -279,6 +287,12 @@ public void testFindInvalidField() {
         insert();
         DocumentCursor cursor = collection.find(where("myField").eq("myData"));
         assertEquals(cursor.size(), 0);
+
+        cursor = collection.find(where("myField").notEq(null));
+        assertEquals(cursor.size(), 0);
+
+        cursor = collection.find(where("myField").eq(null));
+        assertEquals(cursor.size(), 3);
     }
 
     @Test
@@ -303,6 +317,9 @@ public void testGetById() {
         assertNull(document);
 
         document = collection.find().firstOrNull();
+        id = document.getId();
+
+        document = collection.getById(id);
 
         assertEquals(document.get(DOC_ID), document.getId().getIdValue());
         assertEquals(document.get("firstName"), "fn1");
@@ -358,6 +375,42 @@ public void testProject() {
         assertEquals(iteration, 3);
     }
 
+    @Test
+    public void testProjection() {
+        Document doc1 = Document.createDocument("name", "John")
+            .put("address", Document.createDocument("street", "Main Street")
+                .put("city", "New York")
+                .put("state", "NY")
+                .put("zip", "10001"));
+
+        Document doc2 = Document.createDocument("name", "Jane")
+            .put("address", Document.createDocument("street", "Other Street")
+                .put("city", "New Jersey")
+                .put("state", "NJ")
+                .put("zip", "70001"));
+
+        NitriteCollection collection = db.getCollection("person");
+        collection.insert(doc1, doc2);
+
+        StringFieldEncryptionProcessor processor = new StringFieldEncryptionProcessor("pass");
+        processor.addFields("name");
+        processor.process(collection);
+        collection.addProcessor(processor);
+
+        Document projection = Document.createDocument("name", null)
+            .put("address.city", null)
+            .put("address.state", null);
+
+        RecordStream recordStream = collection.find().project(projection);
+        assertEquals(recordStream.size(), 2);
+        assertEquals(recordStream.firstOrNull(), Document.createDocument("name", "John")
+            .put("address", Document.createDocument("city", "New York").put("state", "NY")));
+        assertEquals(recordStream.toList().stream().skip(1).findFirst().orElse(null),
+            Document.createDocument("name", "Jane").put("address", Document.createDocument("city", "New Jersey")
+                .put("state", "NJ")));
+        System.out.println(recordStream.firstOrNull());
+    }
+
     @Test
     public void testProjectWithCustomDocument() {
         insert();
@@ -420,7 +473,7 @@ public void testFindWithIterableEqual() {
             new ArrayList() {{
                 add("three");
                 add("four");
-                add("three");
+                add("five");
             }}));
         assertNotNull(ids);
         assertEquals(ids.size(), 1);
@@ -729,7 +782,7 @@ public void testIdSet() {
         assertEquals(cursor.size(), 1);
 
         Document byId = cursor.iterator().next();
-        assertEquals(byId.get("lastName"), "ln1");
+        assertEquals(collection.getById(byId.getId()).get("lastName"), "ln1");
     }
 
     @Test
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java
index 84fd453ac..0fabd00c6 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionIndexTest.java
@@ -130,7 +130,7 @@ public void testRebuildIndex() {
         insert();
         Collection indices = collection.listIndices();
         for (IndexDescriptor idx : indices) {
-            collection.rebuildIndex(idx.getIndexFields().getFieldNames().toArray(new String[0]));
+            collection.rebuildIndex(idx.getFields().getFieldNames().toArray(new String[0]));
         }
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java
index fe37845e1..a59e7e459 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionSingleFieldIndexTest.java
@@ -134,7 +134,7 @@ public void testRebuildIndex() {
         insert();
         Collection indices = collection.listIndices();
         for (IndexDescriptor idx : indices) {
-            collection.rebuildIndex(idx.getIndexFields().getFieldNames().toArray(new String[0]));
+            collection.rebuildIndex(idx.getFields().getFieldNames().toArray(new String[0]));
         }
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionTest.java
index 4da9d6ae6..abdfc0d17 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionTest.java
@@ -23,6 +23,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author Anindya Chatterjee
@@ -38,7 +39,7 @@ public void testGetName() {
     public void testDropCollection() {
         // check if collection exists
         // the collection is noty opened yet
-        db.hasCollection("test");
+        assertTrue(db.hasCollection("test"));
 
         // destroy the collection
         collection.drop();
@@ -50,9 +51,9 @@ public void testDropCollection() {
     }
 
     @Test
-    public void testCloseConnection() {
+    public void testCloseCollection() {
+        assertTrue(collection.isOpen());
         collection.close();
-
         assertFalse(collection.isOpen());
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java
index 0055f97ac..08cf8fc9f 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/FieldProcessorTest.java
@@ -88,6 +88,7 @@ public Document processAfterRead(Document document) {
             .put("expiryDate", new Date());
         collection.insert(document);
 
+        cvvProcessor.process(collection);
         collection.addProcessor(cvvProcessor);
     }
 
@@ -198,15 +199,4 @@ public void testIndexOnEncryptedField() {
         Document document = collection.find(where("cvv").eq("008")).firstOrNull();
         assertNull(document);
     }
-
-    @Test
-    public void testRemoveProcessor() {
-        Document document = collection.find(where("cvv").eq("008")).firstOrNull();
-        assertNull(document);
-
-        collection.removeProcessor(cvvProcessor);
-
-        document = collection.find(where("cvv").eq("008")).firstOrNull();
-        assertNotNull(document);
-    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java
index f3ae1b720..2b01b1b19 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/BaseObjectRepositoryTest.java
@@ -19,8 +19,11 @@
 
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.NitriteBuilder;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.integration.repository.data.*;
+import org.dizitart.no2.integration.repository.decorator.*;
+import org.dizitart.no2.integration.transaction.TxData;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.junit.After;
 import org.junit.Before;
@@ -44,6 +47,8 @@ public abstract class BaseObjectRepositoryTest {
     protected ObjectRepository aObjectRepository;
     protected ObjectRepository cObjectRepository;
     protected ObjectRepository bookRepository;
+    protected ObjectRepository productRepository;
+    protected ObjectRepository upcomingProductRepository;
 
     @Rule
     public Retry retry = new Retry(3);
@@ -68,6 +73,9 @@ public void setUp() {
 
         bookRepository = db.getRepository(Book.class);
 
+        productRepository = db.getRepository(new ProductDecorator());
+        upcomingProductRepository = db.getRepository(new ProductDecorator(), "upcoming");
+
         for (int i = 0; i < 10; i++) {
             Company company = DataGenerator.generateCompanyRecord();
             companyRepository.insert(company);
@@ -80,6 +88,12 @@ public void setUp() {
 
             Book book = DataGenerator.randomBook();
             bookRepository.insert(book);
+
+            Product product = DataGenerator.randomProduct();
+            productRepository.insert(product);
+
+            product = DataGenerator.randomProduct();
+            upcomingProductRepository.insert(product);
         }
     }
 
@@ -92,6 +106,30 @@ private void openDb() {
         } else {
             db = nitriteBuilder.openOrCreate();
         }
+
+        SimpleDocumentMapper mapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        mapper.registerEntityConverter(new Company.CompanyConverter());
+        mapper.registerEntityConverter(new Employee.EmployeeConverter());
+        mapper.registerEntityConverter(new Note.NoteConverter());
+        mapper.registerEntityConverter(new Book.BookConverter());
+        mapper.registerEntityConverter(new BookId.BookIdConverter());
+        mapper.registerEntityConverter(new ClassA.ClassAConverter());
+        mapper.registerEntityConverter(new ClassBConverter());
+        mapper.registerEntityConverter(new ClassC.ClassCConverter());
+        mapper.registerEntityConverter(new ElemMatch.Converter());
+        mapper.registerEntityConverter(new InternalClass.Converter());
+        mapper.registerEntityConverter(new UniversalTextTokenizerTest.TextData.Converter());
+        mapper.registerEntityConverter(new SubEmployee.Converter());
+        mapper.registerEntityConverter(new ProductScore.Converter());
+        mapper.registerEntityConverter(new PersonEntity.Converter());
+        mapper.registerEntityConverter(new RepeatableIndexTest.Converter());
+        mapper.registerEntityConverter(new EncryptedPerson.Converter());
+        mapper.registerEntityConverter(new TxData.Converter());
+        mapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter());
+        mapper.registerEntityConverter(new ProductConverter());
+        mapper.registerEntityConverter(new ProductIdConverter());
+        mapper.registerEntityConverter(new ManufacturerConverter());
+        mapper.registerEntityConverter(new MiniProduct.Converter());
     }
 
     @After
@@ -116,6 +154,14 @@ public void clear() throws Exception {
             bookRepository.remove(ALL);
         }
 
+        if (productRepository != null && !productRepository.isDropped()) {
+            productRepository.remove(ALL);
+        }
+
+        if (upcomingProductRepository != null && !upcomingProductRepository.isDropped()) {
+            upcomingProductRepository.remove(ALL);
+        }
+
         if (db != null && !db.isClosed()) {
             db.commit();
             db.close();
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java
index fe768e51a..1920e9c3c 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/CustomFieldSeparatorTest.java
@@ -23,17 +23,18 @@
 import lombok.ToString;
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.NitriteConfig;
-import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.index.IndexType;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
+import org.dizitart.no2.index.IndexType;
+import org.dizitart.no2.integration.Retry;
+import org.dizitart.no2.integration.repository.data.Company;
+import org.dizitart.no2.integration.repository.data.Note;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.annotations.Id;
 import org.dizitart.no2.repository.annotations.Index;
 import org.dizitart.no2.repository.annotations.Indices;
-import org.dizitart.no2.integration.repository.data.Company;
-import org.dizitart.no2.integration.repository.data.Note;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -61,6 +62,11 @@ public void setUp() {
             .fieldSeparator(":")
             .openOrCreate();
 
+        SimpleDocumentMapper mapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        mapper.registerEntityConverter(new Company.CompanyConverter());
+        mapper.registerEntityConverter(new EmployeeForCustomSeparator.EmployeeForCustomSeparatorConverter());
+        mapper.registerEntityConverter(new Note.NoteConverter());
+
         repository = db.getRepository(EmployeeForCustomSeparator.class);
     }
 
@@ -107,11 +113,11 @@ public void testFindByEmbeddedField() {
     @ToString
     @EqualsAndHashCode
     @Indices({
-        @Index(value = "joinDate", type = IndexType.NON_UNIQUE),
-        @Index(value = "address", type = IndexType.FULL_TEXT),
-        @Index(value = "employeeNote:text", type = IndexType.FULL_TEXT)
+        @Index(fields = "joinDate", type = IndexType.NON_UNIQUE),
+        @Index(fields = "address", type = IndexType.FULL_TEXT),
+        @Index(fields = "employeeNote:text", type = IndexType.FULL_TEXT)
     })
-    public static class EmployeeForCustomSeparator implements Serializable, Mappable {
+    public static class EmployeeForCustomSeparator implements Serializable {
         @Id
         @Getter
         @Setter
@@ -149,29 +155,41 @@ public EmployeeForCustomSeparator(EmployeeForCustomSeparator copy) {
             employeeNote = copy.employeeNote;
         }
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return Document.createDocument().put("empId", empId)
-                .put("joinDate", joinDate)
-                .put("address", address)
-                .put("blob", blob)
-                .put("company", company.write(mapper))
-                .put("employeeNote", employeeNote.write(mapper));
-        }
-
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            empId = document.get("empId", Long.class);
-            joinDate = document.get("joinDate", Date.class);
-            address = document.get("address", String.class);
-            blob = document.get("blob", byte[].class);
-            employeeNote = new Note();
-            Document doc = document.get("employeeNote", Document.class);
-            employeeNote.read(mapper, doc);
-            company = new Company();
-            doc = document.get("company", Document.class);
-            company.read(mapper, doc);
+        public static class EmployeeForCustomSeparatorConverter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return EmployeeForCustomSeparator.class;
+            }
+
+            @Override
+            public Document toDocument(EmployeeForCustomSeparator entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument().put("empId", entity.empId)
+                    .put("joinDate", entity.joinDate)
+                    .put("address", entity.address)
+                    .put("blob", entity.blob)
+                    .put("company", nitriteMapper.convert(entity.company, Document.class))
+                    .put("employeeNote", nitriteMapper.convert(entity.employeeNote, Document.class));
+            }
+
+            @Override
+            public EmployeeForCustomSeparator fromDocument(Document document, NitriteMapper nitriteMapper) {
+                EmployeeForCustomSeparator entity = new EmployeeForCustomSeparator();
+
+                entity.empId = document.get("empId", Long.class);
+                entity.joinDate = document.get("joinDate", Date.class);
+                entity.address = document.get("address", String.class);
+                entity.blob = document.get("blob", byte[].class);
+
+                Document doc = document.get("employeeNote", Document.class);
+                entity.employeeNote = nitriteMapper.convert(doc, Note.class);
+
+                doc = document.get("company", Document.class);
+                entity.company = nitriteMapper.convert(doc, Company.class);
+                return entity;
+            }
         }
     }
 
+
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java
index be50bff32..5f5297a50 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/FieldProcessorTest.java
@@ -63,6 +63,10 @@ public void setUp() {
 
         persons.insert(person);
 
+        // process existing data
+        fieldProcessor.process(persons);
+
+        // add for further changes
         persons.addProcessor(fieldProcessor);
 
         person = new EncryptedPerson();
@@ -187,21 +191,4 @@ public void testIndexOnEncryptedField() {
         EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull();
         assertNull(person);
     }
-
-    @Test
-    public void testRemoveProcessor() {
-        EncryptedPerson person = persons.find(where("cvv").eq("008")).firstOrNull();
-        assertNull(person);
-
-        person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull();
-        assertNull(person);
-
-        persons.removeProcessor(fieldProcessor);
-
-        person = persons.find(where("cvv").eq("008")).firstOrNull();
-        assertNotNull(person);
-
-        person = persons.find(where("creditCardNumber").eq("5548960345687452")).firstOrNull();
-        assertNotNull(person);
-    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java
index 1a52708ce..e6a2a58d4 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/InternalClass.java
@@ -19,7 +19,7 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -27,20 +27,29 @@
  * @author Anindya Chatterjee.
  */
 @Data
-class InternalClass implements Mappable {
+class InternalClass {
     @Id
-    private long id;
+    private Long id;
     private String name;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("id", id)
-            .put("name", name);
-    }
+    public static class Converter implements EntityConverter {
+        @Override
+        public Class getEntityType() {
+            return InternalClass.class;
+        }
+
+        @Override
+        public Document toDocument(InternalClass entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("id", entity.id)
+                .put("name", entity.name);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        id = document.get("id", Long.class);
-        name = document.get("name", String.class);
+        @Override
+        public InternalClass fromDocument(Document document, NitriteMapper nitriteMapper) {
+            InternalClass entity = new InternalClass();
+            entity.id = document.get("id", Long.class);
+            entity.name = document.get("name", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java
index cc1036d16..53273f3dc 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/NitriteIdAsIdTest.java
@@ -18,6 +18,7 @@
 package org.dizitart.no2.integration.repository;
 
 import org.dizitart.no2.Nitrite;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.integration.TestUtil;
 import org.dizitart.no2.collection.NitriteId;
@@ -48,6 +49,9 @@ public class NitriteIdAsIdTest {
     @Before
     public void before() {
         db = TestUtil.createDb();
+        SimpleDocumentMapper mapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        mapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter());
+
         repo = db.getRepository(WithNitriteId.class);
     }
 
@@ -84,7 +88,7 @@ public void testNitriteIdField() {
     }
 
     @Test(expected = InvalidIdException.class)
-    public void setIdDuringInsert() {
+    public void testSetIdDuringInsert() {
         WithNitriteId item1 = new WithNitriteId();
         item1.name = "first";
         item1.idField = NitriteId.newId();
@@ -93,7 +97,7 @@ public void setIdDuringInsert() {
     }
 
     @Test
-    public void changeIdDuringUpdate() {
+    public void testChangeIdDuringUpdate() {
         WithNitriteId item2 = new WithNitriteId();
         item2.name = "second";
         WriteResult result = repo.insert(item2);
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java
index 91ced7232..a071dd90b 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryNegativeTest.java
@@ -18,6 +18,7 @@
 package org.dizitart.no2.integration.repository;
 
 import org.dizitart.no2.Nitrite;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.integration.TestUtil;
 import org.dizitart.no2.collection.NitriteId;
@@ -45,6 +46,18 @@ public class ObjectRepositoryNegativeTest {
     @Before
     public void setUp() {
         db = TestUtil.createDb();
+        SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        documentMapper.registerEntityConverter(new WithPublicField.Converter());
+        documentMapper.registerEntityConverter(new WithObjectId.Converter());
+        documentMapper.registerEntityConverter(new WithOutId.Converter());
+        documentMapper.registerEntityConverter(new WithoutEmbeddedId.Converter());
+        documentMapper.registerEntityConverter(new WithoutEmbeddedId.NestedId.Converter());
+        documentMapper.registerEntityConverter(new WithEmptyStringId.Converter());
+        documentMapper.registerEntityConverter(new WithNullId.Converter());
+        documentMapper.registerEntityConverter(new Employee.EmployeeConverter());
+        documentMapper.registerEntityConverter(new Company.CompanyConverter());
+        documentMapper.registerEntityConverter(new Note.NoteConverter());
+        documentMapper.registerEntityConverter(new WithNitriteId.WithNitriteIdConverter());
     }
 
     @After
@@ -53,7 +66,7 @@ public void close() throws Exception {
         db = null;
     }
 
-    @Test(expected = ObjectMappingException.class)
+    @Test(expected = ValidationException.class)
     public void testWithCircularReference() {
         ObjectRepository repository = db.getRepository(WithCircularReference.class);
 
@@ -73,7 +86,7 @@ public void testWithCircularReference() {
         }
     }
 
-    @Test(expected = ObjectMappingException.class)
+    @Test(expected = ValidationException.class)
     public void testWithCustomConstructor() {
         ObjectRepository repository = db.getRepository(WithCustomConstructor.class);
 
@@ -126,18 +139,6 @@ public void testFindResultRemove() {
         result.iterator().remove();
     }
 
-    @Test(expected = IndexingException.class)
-    public void testWithObjectId() {
-        ObjectRepository repository = db.getRepository(WithObjectId.class);
-        WithOutId id = new WithOutId();
-        id.setName("test");
-        id.setNumber(1);
-
-        WithObjectId object = new WithObjectId();
-        object.setWithOutId(id);
-        repository.insert(object);
-    }
-
     @Test(expected = NotIdentifiableException.class)
     public void testUpdateNoId() {
         ObjectRepository repository = db.getRepository(WithOutId.class);
@@ -179,7 +180,7 @@ public void testGetByNullId() {
         ObjectRepository repository = db.getRepository(WithPublicField.class);
         WithPublicField object = new WithPublicField();
         object.name = "test";
-        object.number = 2;
+        object.number = 2L;
 
         repository.insert(object);
         WithPublicField instance = repository.getById(null);
@@ -215,7 +216,7 @@ public void testGetByWrongIdType() {
         ObjectRepository repository = db.getRepository(WithPublicField.class);
         WithPublicField object = new WithPublicField();
         object.name = "test";
-        object.number = 2;
+        object.number = 2L;
 
         repository.insert(object);
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java
index 0443fadae..98759627e 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/ObjectRepositoryTest.java
@@ -19,17 +19,21 @@
 
 import com.github.javafaker.Faker;
 import lombok.Data;
+
+import org.apache.commons.lang3.time.StopWatch;
 import org.dizitart.no2.Nitrite;
-import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteCollection;
-import org.dizitart.no2.collection.meta.Attributes;
-import org.dizitart.no2.common.mapper.Mappable;
-import org.dizitart.no2.common.mapper.MappableMapper;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
+import org.dizitart.no2.common.meta.Attributes;
+import org.dizitart.no2.exceptions.UniqueConstraintException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.dizitart.no2.index.IndexType;
+import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.integration.repository.data.*;
+import org.dizitart.no2.integration.repository.decorator.*;
 import org.dizitart.no2.repository.Cursor;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.annotations.Entity;
@@ -61,12 +65,29 @@ public class ObjectRepositoryTest {
 
     @Before
     public void setUp() {
-        NitriteMapper mapper = new MappableMapper();
+        SimpleDocumentMapper mapper = new SimpleDocumentMapper();
+        mapper.registerEntityConverter(new InternalClass.Converter());
+        mapper.registerEntityConverter(new EmployeeEntity.Converter());
+        mapper.registerEntityConverter(new StressRecord.Converter());
+        mapper.registerEntityConverter(new WithClassField.Converter());
+        mapper.registerEntityConverter(new WithDateId.Converter());
+        mapper.registerEntityConverter(new WithTransientField.Converter());
+        mapper.registerEntityConverter(new WithOutId.Converter());
+        mapper.registerEntityConverter(new ChildClass.Converter());
+        mapper.registerEntityConverter(new WithOutGetterSetter.Converter());
+        mapper.registerEntityConverter(new WithPrivateConstructor.Converter());
+        mapper.registerEntityConverter(new WithPublicField.Converter());
+        mapper.registerEntityConverter(new Employee.EmployeeConverter());
+        mapper.registerEntityConverter(new Company.CompanyConverter());
+        mapper.registerEntityConverter(new ProductConverter());
+        mapper.registerEntityConverter(new ProductIdConverter());
+        mapper.registerEntityConverter(new ManufacturerConverter());
+        mapper.registerEntityConverter(new MiniProduct.Converter());
 
         db = Nitrite.builder()
-            .loadModule(module(mapper))
-            .fieldSeparator(".")
-            .openOrCreate();
+                .loadModule(module(mapper))
+                .fieldSeparator(".")
+                .openOrCreate();
     }
 
     @After
@@ -105,7 +126,7 @@ public void testWithOutId() {
         ObjectRepository repository = db.getRepository(WithOutId.class);
         WithOutId object = new WithOutId();
         object.setName("test");
-        object.setNumber(2);
+        object.setNumber(2L);
 
         repository.insert(object);
         for (WithOutId instance : repository.find()) {
@@ -119,7 +140,7 @@ public void testWithPublicField() {
         ObjectRepository repository = db.getRepository(WithPublicField.class);
         WithPublicField object = new WithPublicField();
         object.name = "test";
-        object.number = 2;
+        object.number = 2L;
 
         repository.insert(object);
         WithPublicField instance = repository.getById("test");
@@ -131,7 +152,7 @@ public void testWithPublicField() {
     public void testWithTransientField() {
         ObjectRepository repository = db.getRepository(WithTransientField.class);
         WithTransientField object = new WithTransientField();
-        object.setNumber(2);
+        object.setNumber(2L);
         object.setName("test");
 
         repository.insert(object);
@@ -145,6 +166,8 @@ public void testWithTransientField() {
     public void testWriteThousandRecords() {
         int count = 5000;
 
+        StopWatch sw = new StopWatch();
+        sw.start();
         ObjectRepository repository = db.getRepository(StressRecord.class);
 
         for (int i = 0; i < count; i++) {
@@ -157,19 +180,20 @@ public void testWriteThousandRecords() {
             repository.insert(record);
         }
 
-        Cursor cursor
-            = repository.find(where("failed").eq(false));
+        Cursor cursor = repository.find(where("failed").eq(false));
         for (StressRecord record : cursor) {
             record.setProcessed(true);
             repository.update(where("firstName").eq(record.getFirstName()), record);
         }
+        sw.stop();
+        System.out.println("Sequential Time (s) - " + sw.getTime());
     }
 
     @Test
     public void testWithPackagePrivateClass() {
         ObjectRepository repository = db.getRepository(InternalClass.class);
         InternalClass internalClass = new InternalClass();
-        internalClass.setId(1);
+        internalClass.setId(1L);
         internalClass.setName("name");
 
         repository.insert(internalClass);
@@ -180,8 +204,7 @@ public void testWithPackagePrivateClass() {
 
     @Test
     public void testWithPrivateConstructor() {
-        ObjectRepository repository =
-            db.getRepository(WithPrivateConstructor.class);
+        ObjectRepository repository = db.getRepository(WithPrivateConstructor.class);
 
         WithPrivateConstructor object = WithPrivateConstructor.create("test", 2L);
         repository.insert(object);
@@ -205,9 +228,9 @@ public void testWithDateAsId() {
         repository.insert(object2);
 
         assertEquals(repository.find(where("id").eq(new Date(1482773634L)))
-            .firstOrNull(), object1);
+                .firstOrNull(), object1);
         assertEquals(repository.find(where("id").eq(new Date(1482773720L)))
-            .firstOrNull(), object2);
+                .firstOrNull(), object2);
     }
 
     @Test
@@ -225,7 +248,7 @@ public void testWithIdInheritance() {
         repository.insert(childClass);
 
         childClass = new ChildClass();
-        childClass.setName("seconds");
+        childClass.setName("second");
         childClass.setDate(new Date(100001L));
         childClass.setId(2L);
         childClass.setText("I am second class");
@@ -282,7 +305,7 @@ public void testKeyedRepository() {
         assertTrue(db.hasRepository(Employee.class, "developers"));
 
         assertEquals(db.listRepositories().size(), 1);
-        assertEquals(db.listKeyedRepository().size(), 2);
+        assertEquals(db.listKeyedRepositories().size(), 2);
 
         assertEquals(employeeRepo.find(where("address").text("abcd")).size(), 1);
         assertEquals(employeeRepo.find(where("address").text("xyz")).size(), 1);
@@ -311,16 +334,16 @@ public void testEntityRepository() {
         assertTrue(errored);
 
         assertTrue(db.listRepositories().contains("entity.employee"));
-        assertEquals(db.listKeyedRepository().size(), 2);
+        assertEquals(db.listKeyedRepositories().size(), 2);
         assertEquals(db.listCollectionNames().size(), 0);
 
         assertTrue(managerRepo.hasIndex("firstName"));
         assertTrue(managerRepo.hasIndex("lastName"));
-        assertTrue(employeeRepo.hasIndex("lastName"));
+        assertTrue(employeeRepo.hasIndex("firstName"));
         assertTrue(employeeRepo.hasIndex("lastName"));
 
         managerRepo.drop();
-        assertEquals(db.listKeyedRepository().size(), 1);
+        assertEquals(db.listKeyedRepositories().size(), 1);
     }
 
     @Test
@@ -334,12 +357,177 @@ public void testIssue217() {
         await().atMost(5, TimeUnit.SECONDS).until(() -> counter.get() == 1);
     }
 
+    @Test
+    public void testRepositoryName() {
+        ObjectRepository productRepository = db.getRepository(new ProductDecorator());
+        ObjectRepository upcomingProductRepository = db.getRepository(new ProductDecorator(), "upcoming");
+        ObjectRepository manufacturerRepository = db.getRepository(new ManufacturerDecorator());
+        ObjectRepository exManufacturerRepository = db.getRepository(new ManufacturerDecorator(), "ex");
+        ObjectRepository employeeRepository = db.getRepository(Employee.class);
+        ObjectRepository managerRepository = db.getRepository(Employee.class, "manager");
+
+        assertEquals(productRepository.getDocumentCollection().getName(), "product");
+        assertEquals(upcomingProductRepository.getDocumentCollection().getName(),
+                "product+upcoming");
+        assertEquals(manufacturerRepository.getDocumentCollection().getName(), Manufacturer.class.getName());
+        assertEquals(exManufacturerRepository.getDocumentCollection().getName(), Manufacturer.class.getName() + "+ex");
+        assertEquals(employeeRepository.getDocumentCollection().getName(), Employee.class.getName());
+        assertEquals(managerRepository.getDocumentCollection().getName(), Employee.class.getName() + "+manager");
+    }
+
+    @Test
+    public void testRepositoryType() {
+        ObjectRepository productRepository = db.getRepository(new ProductDecorator());
+        ObjectRepository upcomingProductRepository = db.getRepository(new ProductDecorator(), "upcoming");
+        ObjectRepository manufacturerRepository = db.getRepository(new ManufacturerDecorator());
+        ObjectRepository exManufacturerRepository = db.getRepository(new ManufacturerDecorator(), "ex");
+        ObjectRepository employeeRepository = db.getRepository(Employee.class);
+        ObjectRepository managerRepository = db.getRepository(Employee.class, "manager");
+
+        assertEquals(productRepository.getType(), Product.class);
+        assertEquals(upcomingProductRepository.getType(), Product.class);
+        assertEquals(manufacturerRepository.getType(), Manufacturer.class);
+        assertEquals(exManufacturerRepository.getType(), Manufacturer.class);
+        assertEquals(employeeRepository.getType(), Employee.class);
+        assertEquals(managerRepository.getType(), Employee.class);
+    }
+
+    @Test
+    public void testDestroyRepository() {
+        ObjectRepository productRepository = db.getRepository(new ProductDecorator());
+        ObjectRepository upcomingProductRepository = db.getRepository(new ProductDecorator(), "upcoming");
+        ObjectRepository manufacturerRepository = db.getRepository(new ManufacturerDecorator());
+        ObjectRepository exManufacturerRepository = db.getRepository(new ManufacturerDecorator(), "ex");
+        ObjectRepository employeeRepository = db.getRepository(Employee.class);
+        ObjectRepository managerRepository = db.getRepository(Employee.class, "manager");
+
+        assertNotNull(productRepository);
+        assertNotNull(upcomingProductRepository);
+        assertNotNull(manufacturerRepository);
+        assertNotNull(exManufacturerRepository);
+        assertNotNull(employeeRepository);
+        assertNotNull(managerRepository);
+
+        assertTrue(db.hasRepository(new ProductDecorator()));
+        assertTrue(db.hasRepository(new ProductDecorator(), "upcoming"));
+        assertTrue(db.hasRepository(new ManufacturerDecorator()));
+        assertTrue(db.hasRepository(new ManufacturerDecorator(), "ex"));
+        assertTrue(db.hasRepository(Employee.class));
+        assertTrue(db.hasRepository(Employee.class, "manager"));
+
+        db.destroyRepository(new ProductDecorator());
+        assertFalse(db.hasRepository(new ProductDecorator()));
+        assertTrue(db.hasRepository(new ProductDecorator(), "upcoming"));
+
+        db.destroyRepository(new ProductDecorator(), "upcoming");
+        assertFalse(db.hasRepository(new ProductDecorator()));
+        assertFalse(db.hasRepository(new ProductDecorator(), "upcoming"));
+
+        db.destroyRepository(new ManufacturerDecorator());
+        db.destroyRepository(new ManufacturerDecorator(), "ex");
+        assertFalse(db.hasRepository(new ManufacturerDecorator()));
+        assertFalse(db.hasRepository(new ManufacturerDecorator(), "ex"));
+
+        db.destroyRepository(Employee.class);
+        assertFalse(db.hasRepository(Employee.class));
+        assertTrue(db.hasRepository(Employee.class, "manager"));
+
+        db.destroyRepository(Employee.class, "manager");
+        assertFalse(db.hasRepository(Employee.class));
+        assertFalse(db.hasRepository(Employee.class, "manager"));
+    }
+
+    @Test
+    public void testDestroyRepositoryWrongDecorator() {
+        ObjectRepository productRepository = db.getRepository(new ProductDecorator());
+        assertNotNull(productRepository);
+        assertTrue(db.hasRepository(new ProductDecorator()));
+        assertFalse(db.hasRepository(new ManufacturerDecorator()));
+
+        db.destroyRepository(new ManufacturerDecorator());
+        assertTrue(db.hasRepository(new ProductDecorator()));
+        assertFalse(db.hasRepository(new ManufacturerDecorator()));
+    }
+
+    @Test
+    public void testDestroyRepositoryWrongDecoratorWithKey() {
+        ObjectRepository productRepository = db.getRepository(new ProductDecorator(), "upcoming");
+        assertNotNull(productRepository);
+        assertTrue(db.hasRepository(new ProductDecorator(), "upcoming"));
+        assertFalse(db.hasRepository(new ManufacturerDecorator()));
+
+        db.destroyRepository(new ManufacturerDecorator(), "upcoming");
+        assertTrue(db.hasRepository(new ProductDecorator(), "upcoming"));
+        assertFalse(db.hasRepository(new ManufacturerDecorator()));
+    }
+
+    @Test
+    public void testDestroyRepositoryWrongClassName() {
+        ObjectRepository productRepository = db.getRepository(new ProductDecorator());
+        assertNotNull(productRepository);
+        assertFalse(db.hasRepository(Product.class));
+
+        db.destroyRepository(Product.class);
+        assertTrue(db.hasRepository(new ProductDecorator()));
+        assertFalse(db.hasRepository(Product.class));
+    }
+
+    @Test
+    public void testDestroyRepositoryWrongClassNameAndKey() {
+        ObjectRepository employeeRepository = db.getRepository(Employee.class);
+        assertNotNull(employeeRepository);
+        assertTrue(db.hasRepository(Employee.class));
+
+        db.destroyRepository(Employee.class, "manager");
+        assertTrue(db.hasRepository(Employee.class));
+        assertFalse(db.hasRepository(Employee.class, "manager"));
+    }
+
+    @Test
+    public void testHasRepository() {
+        ObjectRepository employeeRepository = db.getRepository(Employee.class);
+        ObjectRepository productRepository = db.getRepository(new ProductDecorator());
+
+        assertNotNull(employeeRepository);
+        assertNotNull(productRepository);
+
+        assertTrue(db.hasRepository(Employee.class));
+        assertFalse(db.hasRepository(Employee.class, "manager"));
+        assertTrue(db.hasRepository(new ProductDecorator()));
+        assertFalse(db.hasRepository(new ProductDecorator(), "ex"));
+    }
+
+    @Test
+    public void testIssue767() {
+        ObjectRepository companyRepository = db.getRepository(Company.class);
+        Company company1 = new Company();
+        company1.setCompanyId(1L);
+        company1.setCompanyName("ABCD");
+        company1.setDateCreated(new Date());
+        companyRepository.insert(company1);
+
+        Company company2 = new Company();
+        company2.setCompanyId(2L);
+        company2.setCompanyName("ABCD");
+
+        boolean uniqueConstraintError = false;
+        try {
+            companyRepository.insert(company2);
+        } catch (UniqueConstraintException e) {
+            uniqueConstraintError = true;
+        } finally {
+            assertTrue(uniqueConstraintError);
+        }
+
+        assertEquals(companyRepository.find().size(), 1);
+    }
+
     @Data
     @Entity(value = "entity.employee", indices = {
-        @Index(value = "firstName", type = IndexType.NON_UNIQUE),
-        @Index(value = "lastName", type = IndexType.NON_UNIQUE),
+            @Index(fields = "firstName", type = IndexType.NON_UNIQUE),
+            @Index(fields = "lastName", type = IndexType.NON_UNIQUE),
     })
-    private static class EmployeeEntity implements Mappable {
+    private static class EmployeeEntity {
         private static final Faker faker = new Faker();
 
         @Id
@@ -353,18 +541,28 @@ public EmployeeEntity() {
             lastName = faker.name().lastName();
         }
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return Document.createDocument("id", id)
-                .put("firstName", firstName)
-                .put("lastName", lastName);
-        }
-
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            id = document.get("id", Long.class);
-            firstName = document.get("firstName", String.class);
-            lastName = document.get("lastName", String.class);
+        public static class Converter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return EmployeeEntity.class;
+            }
+
+            @Override
+            public Document toDocument(EmployeeEntity entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument("id", entity.id)
+                        .put("firstName", entity.firstName)
+                        .put("lastName", entity.lastName);
+            }
+
+            @Override
+            public EmployeeEntity fromDocument(Document document, NitriteMapper nitriteMapper) {
+                EmployeeEntity entity = new EmployeeEntity();
+                entity.id = document.get("id", Long.class);
+                entity.firstName = document.get("firstName", String.class);
+                entity.lastName = document.get("lastName", String.class);
+                return entity;
+            }
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java
index d29b4cf22..46dcdbcb0 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryFactoryTest.java
@@ -22,7 +22,7 @@
 import org.dizitart.no2.integration.TestUtil;
 import org.dizitart.no2.collection.*;
 import org.dizitart.no2.collection.events.CollectionEventListener;
-import org.dizitart.no2.collection.meta.Attributes;
+import org.dizitart.no2.common.meta.Attributes;
 import org.dizitart.no2.common.WriteResult;
 import org.dizitart.no2.common.concurrent.LockService;
 import org.dizitart.no2.exceptions.ValidationException;
@@ -59,10 +59,10 @@ public void testRepositoryFactory() {
     public void testNullType() {
         RepositoryFactory factory = new RepositoryFactory(new CollectionFactory(new LockService()));
         db = TestUtil.createDb();
-        factory.getRepository(db.getConfig(), null, "dummy");
+        factory.getRepository(db.getConfig(), (Class) null, "dummy");
     }
 
-    @Test
+    @Test(expected = ValidationException.class)
     public void testNullCollection() {
         RepositoryFactory factory = new RepositoryFactory(new CollectionFactory(new LockService()));
         db = TestUtil.createDb();
@@ -129,11 +129,6 @@ public void addProcessor(Processor processor) {
 
         }
 
-        @Override
-        public void removeProcessor(Processor processor) {
-
-        }
-
         @Override
         public void createIndex(IndexOptions indexOptions, String... fields) {
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java
index 84b60956d..8f8d3a697 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryJoinTest.java
@@ -20,14 +20,15 @@
 import lombok.Data;
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.NitriteBuilder;
-import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.Lookup;
 import org.dizitart.no2.common.RecordStream;
-import org.dizitart.no2.exceptions.InvalidOperationException;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
+import org.dizitart.no2.exceptions.InvalidOperationException;
+import org.dizitart.no2.integration.Retry;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.annotations.Id;
 import org.junit.After;
@@ -103,6 +104,11 @@ private void openDb() {
         } else {
             db = nitriteBuilder.openOrCreate();
         }
+
+        SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        documentMapper.registerEntityConverter(new Person.Converter());
+        documentMapper.registerEntityConverter(new Address.Converter());
+        documentMapper.registerEntityConverter(new PersonDetails.Converter());
     }
 
     @After
@@ -171,80 +177,120 @@ public void testRemove() {
     }
 
     @Data
-    public static class Person implements Mappable {
+    public static class Person {
         @Id
         private NitriteId nitriteId;
         private String id;
         private String name;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return createDocument()
-                .put("nitriteId", nitriteId)
-                .put("id", id)
-                .put("name", name);
-        }
+        public static class Converter implements EntityConverter {
 
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            nitriteId = document.get("nitriteId", NitriteId.class);
-            id = document.get("id", String.class);
-            name = document.get("name", String.class);
+            @Override
+            public Class getEntityType() {
+                return Person.class;
+            }
+
+            @Override
+            public Document toDocument(Person entity, NitriteMapper nitriteMapper) {
+                return createDocument()
+                    .put("nitriteId", entity.nitriteId)
+                    .put("id", entity.id)
+                    .put("name", entity.name);
+            }
+
+            @Override
+            public Person fromDocument(Document document, NitriteMapper nitriteMapper) {
+                Person entity = new Person();
+                entity.nitriteId = document.get("nitriteId", NitriteId.class);
+                entity.id = document.get("id", String.class);
+                entity.name = document.get("name", String.class);
+                return entity;
+            }
         }
     }
 
     @Data
-    public static class Address implements Mappable {
+    public static class Address {
         @Id
         private NitriteId nitriteId;
         private String personId;
         private String street;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return createDocument()
-                .put("nitriteId", nitriteId)
-                .put("personId", personId)
-                .put("street", street);
-        }
+        public static class Converter implements EntityConverter
 {
 
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            nitriteId = document.get("nitriteId", NitriteId.class);
-            personId = document.get("personId", String.class);
-            street = document.get("street", String.class);
+            @Override
+            public Class
 getEntityType() {
+                return Address.class;
+            }
+
+            @Override
+            public Document toDocument(Address entity, NitriteMapper nitriteMapper) {
+                return createDocument()
+                    .put("nitriteId", entity.nitriteId)
+                    .put("personId", entity.personId)
+                    .put("street", entity.street);
+            }
+
+            @Override
+            public Address fromDocument(Document document, NitriteMapper nitriteMapper) {
+                Address entity = new Address();
+                entity.nitriteId = document.get("nitriteId", NitriteId.class);
+                entity.personId = document.get("personId", String.class);
+                entity.street = document.get("street", String.class);
+                return entity;
+            }
         }
     }
 
     @Data
-    public static class PersonDetails implements Mappable {
+    public static class PersonDetails {
         @Id
         private NitriteId nitriteId;
         private String id;
         private String name;
         private List
 addresses;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return createDocument()
-                .put("nitriteId", nitriteId)
-                .put("personId", id)
-                .put("street", name)
-                .put("addresses", addresses);
-        }
+        public static class Converter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return PersonDetails.class;
+            }
+
+            @Override
+            public Document toDocument(PersonDetails entity, NitriteMapper nitriteMapper) {
+                List documents = new ArrayList<>();
+                if (entity.addresses != null) {
+                    for (Address address : entity.addresses) {
+                        documents.add(nitriteMapper.convert(address, Document.class));
+                    }
+                }
+
+                return createDocument()
+                    .put("nitriteId", entity.nitriteId)
+                    .put("personId", entity.id)
+                    .put("street", entity.name)
+                    .put("addresses", documents);
+            }
+
+            @Override
+            public PersonDetails fromDocument(Document document, NitriteMapper nitriteMapper) {
+                PersonDetails entity = new PersonDetails();
+
+                entity.nitriteId = document.get("nitriteId", NitriteId.class);
+                entity.id = document.get("id", String.class);
+                entity.name = document.get("name", String.class);
+
+                Collection documents = document.get("addresses", Collection.class);
+                if (documents != null) {
+                    entity.addresses = new ArrayList<>();
+                    for (Document doc : documents) {
+                        Address address = nitriteMapper.convert(doc, Address.class);
+                        entity.addresses.add(address);
+                    }
+                }
 
-        @Override
-        @SuppressWarnings("unchecked")
-        public void read(NitriteMapper mapper, Document document) {
-            nitriteId = document.get("nitriteId", NitriteId.class);
-            id = document.get("id", String.class);
-            name = document.get("name", String.class);
-            Set documents = document.get("addresses", Set.class);
-            this.addresses = new ArrayList<>();
-            for (Document doc : documents) {
-                Address address = new Address();
-                address.read(mapper, doc);
-                addresses.add(address);
+                return entity;
             }
         }
     }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java
index 90559fea1..f195d1a9d 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositoryModificationTest.java
@@ -39,6 +39,7 @@
 
 import static org.awaitility.Awaitility.await;
 import static org.dizitart.no2.collection.Document.createDocument;
+import static org.dizitart.no2.collection.UpdateOptions.updateOptions;
 import static org.dizitart.no2.filters.FluentFilter.where;
 import static org.junit.Assert.*;
 
@@ -210,7 +211,7 @@ public void testUpsertTrue() {
         employee.setEmployeeNote(empNote1);
 
         WriteResult writeResult
-            = employeeRepository.update(where("empId").eq(12), employee, true);
+            = employeeRepository.update(where("empId").eq(12), employee, updateOptions(true));
         assertEquals(writeResult.getAffectedCount(), 1);
 
         result = employeeRepository.find(where("joinDate").eq(joiningDate));
@@ -235,7 +236,7 @@ public void testUpsertFalse() {
         employee.setEmployeeNote(empNote1);
 
         WriteResult writeResult
-            = employeeRepository.update(where("empId").eq(12), employee, false);
+            = employeeRepository.update(where("empId").eq(12), employee, updateOptions(false));
         assertEquals(writeResult.getAffectedCount(), 0);
 
         result = employeeRepository.find(where("joinDate").eq(joiningDate));
@@ -327,7 +328,7 @@ public void testMultiUpdateWithObject() {
         update.setAddress("new address");
 
         WriteResult writeResult
-            = employeeRepository.update(where("joinDate").eq(now), update, false);
+            = employeeRepository.update(where("joinDate").eq(now), update, updateOptions(false));
         assertEquals(writeResult.getAffectedCount(), 0);
     }
 
@@ -361,7 +362,7 @@ public void testUpdateWithChangedId() {
         Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull();
         assertNotNull(result.getJoinDate());
 
-        WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false);
+        WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false));
         assertEquals(writeResult.getAffectedCount(), 1);
 
         assertEquals(count, employeeRepository.size());
@@ -380,7 +381,7 @@ public void testUpdateWithNullId() {
         Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull();
         assertNotNull(result.getJoinDate());
 
-        WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false);
+        WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false));
         assertEquals(writeResult.getAffectedCount(), 1);
     }
 
@@ -396,7 +397,7 @@ public void testUpdateWithDuplicateId() {
         Employee result = employeeRepository.find(where("empId").eq(oldId)).firstOrNull();
         assertNotNull(result.getJoinDate());
 
-        WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, false);
+        WriteResult writeResult = employeeRepository.update(where("empId").eq(oldId), newEmployee, updateOptions(false));
         assertEquals(writeResult.getAffectedCount(), 1);
 
         assertEquals(count, employeeRepository.size());
@@ -550,7 +551,7 @@ public void testDelete() {
     /*
      * Upsert Use Cases
      *
-     * 1. Object does not exists
+     * 1. Object does not exist
      *      a. if upsert true, it will insert
      *      b. if upsert false, nothing happens
      * 2. Object exists
@@ -563,12 +564,12 @@ public void testDelete() {
     public void testUpdateObjectNotExistsUpsertTrue() {
         ObjectRepository repo = db.getRepository(InternalClass.class);
         InternalClass a = new InternalClass();
-        a.setId(1);
+        a.setId(1L);
         a.setName("first");
         repo.insert(a);
 
         a = new InternalClass();
-        a.setId(2);
+        a.setId(2L);
         a.setName("second");
 
         // it will insert as new object
@@ -580,18 +581,18 @@ public void testUpdateObjectNotExistsUpsertTrue() {
     public void testUpdateObjectNotExistsUpsertFalse() {
         ObjectRepository repo = db.getRepository(InternalClass.class);
         InternalClass a = new InternalClass();
-        a.setId(1);
+        a.setId(1L);
         a.setName("first");
         repo.insert(a);
 
         a = new InternalClass();
-        a.setId(2);
+        a.setId(2L);
         a.setName("second");
 
         // no changes will happen to repository
         repo.update(a, false);
         assertEquals(repo.size(), 1);
-        assertEquals(repo.find().firstOrNull().getId(), 1);
+        assertEquals(repo.find().firstOrNull().getId().longValue(), 1L);
         assertEquals(repo.find().firstOrNull().getName(), "first");
     }
 
@@ -599,18 +600,18 @@ public void testUpdateObjectNotExistsUpsertFalse() {
     public void testUpdateObjectExistsUpsertTrue() {
         ObjectRepository repo = db.getRepository(InternalClass.class);
         InternalClass a = new InternalClass();
-        a.setId(1);
+        a.setId(1L);
         a.setName("first");
         repo.insert(a);
 
         a = new InternalClass();
-        a.setId(1);
+        a.setId(1L);
         a.setName("second");
 
         // update existing object, keep id same
         repo.update(a, true);
         assertEquals(repo.size(), 1);
-        assertEquals(repo.find().firstOrNull().getId(), 1);
+        assertEquals(repo.find().firstOrNull().getId().longValue(), 1L);
         assertEquals(repo.find().firstOrNull().getName(), "second");
     }
 
@@ -618,18 +619,18 @@ public void testUpdateObjectExistsUpsertTrue() {
     public void testUpdateObjectExistsUpsertFalse() {
         ObjectRepository repo = db.getRepository(InternalClass.class);
         InternalClass a = new InternalClass();
-        a.setId(1);
+        a.setId(1L);
         a.setName("first");
         repo.insert(a);
 
         a = new InternalClass();
-        a.setId(1);
+        a.setId(1L);
         a.setName("second");
 
         // update existing object, keep id same
         repo.update(a, false);
         assertEquals(repo.size(), 1);
-        assertEquals(repo.find().firstOrNull().getId(), 1);
+        assertEquals(repo.find().firstOrNull().getId().longValue(), 1L);
         assertEquals(repo.find().firstOrNull().getName(), "second");
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java
index 29524a7cf..70f1b39d7 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/RepositorySearchTest.java
@@ -20,8 +20,9 @@
 import lombok.Getter;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.common.SortOrder;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.exceptions.FilterException;
 import org.dizitart.no2.exceptions.InvalidIdException;
 import org.dizitart.no2.exceptions.NotIdentifiableException;
@@ -318,7 +319,7 @@ public void testRegexFilter() {
         int count = employees.toList().size();
 
         List employeeList = employeeRepository.find(where("emailAddress")
-            .regex("^[a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+$"))
+                .regex("^[a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+$"))
             .toList();
 
         assertEquals(employeeList.size(), count);
@@ -364,21 +365,20 @@ public void testElemMatchFilter() {
         final ProductScore score6 = new ProductScore("xyz", 8);
 
         ObjectRepository repository = db.getRepository(ElemMatch.class);
-        ElemMatch e1 = new ElemMatch() {{
-            setId(1);
-            setStrArray(new String[]{"a", "b"});
-            setProductScores(new ProductScore[]{score1, score4});
-        }};
-        ElemMatch e2 = new ElemMatch() {{
-            setId(2);
-            setStrArray(new String[]{"d", "e"});
-            setProductScores(new ProductScore[]{score2, score5});
-        }};
-        ElemMatch e3 = new ElemMatch() {{
-            setId(3);
-            setStrArray(new String[]{"a", "f"});
-            setProductScores(new ProductScore[]{score3, score6});
-        }};
+        ElemMatch e1 = new ElemMatch();
+        e1.setId(1L);
+        e1.setStrArray(new String[]{"a", "b"});
+        e1.setProductScores(new ProductScore[]{score1, score4});
+
+        ElemMatch e2 = new ElemMatch();
+        e2.setId(2L);
+        e2.setStrArray(new String[]{"d", "e"});
+        e2.setProductScores(new ProductScore[]{score2, score5});
+
+        ElemMatch e3 = new ElemMatch();
+        e3.setId(3L);
+        e3.setStrArray(new String[]{"a", "f"});
+        e3.setProductScores(new ProductScore[]{score3, score6});
 
         repository.insert(e1, e2, e3);
 
@@ -563,24 +563,37 @@ public void testIdSet() {
     @Test
     public void testBetweenFilter() {
         @Getter
-        class TestData implements Mappable {
+        class TestData {
             private Date age;
 
             public TestData(Date age) {
                 this.age = age;
             }
+        }
+
+        class Converter implements EntityConverter {
+
+            @Override
+            public Class getEntityType() {
+                return TestData.class;
+            }
 
             @Override
-            public Document write(NitriteMapper mapper) {
-                return Document.createDocument("age", age);
+            public Document toDocument(TestData entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument("age", entity.age);
             }
 
             @Override
-            public void read(NitriteMapper mapper, Document document) {
-                age = document.get("age", Date.class);
+            public TestData fromDocument(Document document, NitriteMapper nitriteMapper) {
+                TestData entity = new TestData(new Date());
+                entity.age = document.get("age", Date.class);
+                return entity;
             }
         }
 
+        SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        documentMapper.registerEntityConverter(new Converter());
+
         TestData data1 = new TestData(new GregorianCalendar(2020, Calendar.JANUARY, 11).getTime());
         TestData data2 = new TestData(new GregorianCalendar(2021, Calendar.FEBRUARY, 12).getTime());
         TestData data3 = new TestData(new GregorianCalendar(2022, Calendar.MARCH, 13).getTime());
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/UnAnnotatedObjectTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/UnAnnotatedObjectTest.java
index 5ed5e452d..c5c27490e 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/UnAnnotatedObjectTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/UnAnnotatedObjectTest.java
@@ -18,16 +18,18 @@
 package org.dizitart.no2.integration.repository;
 
 
+import org.dizitart.no2.common.RecordStream;
 import org.dizitart.no2.common.SortOrder;
 import org.dizitart.no2.integration.repository.data.ClassA;
 import org.dizitart.no2.integration.repository.data.ClassC;
+import org.dizitart.no2.integration.repository.decorator.MiniProduct;
+import org.dizitart.no2.integration.repository.decorator.Product;
 import org.dizitart.no2.repository.Cursor;
 import org.junit.Test;
 
 import static org.dizitart.no2.collection.FindOptions.orderBy;
 import static org.dizitart.no2.filters.FluentFilter.where;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.*;
 
 /**
  * @author Anindya Chatterjee.
@@ -74,4 +76,64 @@ public void testFind() {
             System.out.println(classC);
         }
     }
+
+    @Test
+    public void testDecoratedEntityFind() {
+        Cursor cursor = productRepository.find();
+        assertEquals(cursor.size(), 10);
+        assertFalse(cursor.isEmpty());
+
+        assertTrue(productRepository.hasIndex("productId.uniqueId", "productId.productCode"));
+        assertTrue(productRepository.hasIndex("manufacturer.name"));
+        assertTrue(productRepository.hasIndex("productName", "manufacturer.uniqueId"));
+
+        cursor = productRepository.find(where("productId.uniqueId").notEq(null)
+            .and(where("price").gt(0.0)));
+
+        assertTrue(!cursor.isEmpty());
+        assertEquals(cursor.size(), 10);
+
+        RecordStream miniProducts = cursor.project(MiniProduct.class);
+
+        for (MiniProduct miniProduct : miniProducts) {
+            Cursor products = productRepository.find(where("productId.uniqueId")
+                .eq(miniProduct.getUniqueId()));
+
+            assertNotNull(products);
+            assertFalse(products.isEmpty());
+            assertEquals(1, products.size());
+            assertEquals(miniProduct.getManufacturerName(), products.firstOrNull().getManufacturer().getName());
+            assertEquals(miniProduct.getPrice(), products.firstOrNull().getPrice());
+        }
+    }
+
+    @Test
+    public void testDecoratedEntityFindWithTag() {
+        Cursor cursor = upcomingProductRepository.find();
+        assertEquals(cursor.size(), 10);
+        assertFalse(cursor.isEmpty());
+
+        assertTrue(upcomingProductRepository.hasIndex("productId.uniqueId", "productId.productCode"));
+        assertTrue(upcomingProductRepository.hasIndex("manufacturer.name"));
+        assertTrue(upcomingProductRepository.hasIndex("productName", "manufacturer.uniqueId"));
+
+        cursor = upcomingProductRepository.find(where("productId.uniqueId").notEq(null)
+            .and(where("price").gt(0.0)));
+
+        assertTrue(!cursor.isEmpty());
+        assertEquals(cursor.size(), 10);
+
+        RecordStream miniProducts = cursor.project(MiniProduct.class);
+
+        for (MiniProduct miniProduct : miniProducts) {
+            Cursor products = upcomingProductRepository.find(where("productId.uniqueId")
+                .eq(miniProduct.getUniqueId()));
+
+            assertNotNull(products);
+            assertFalse(products.isEmpty());
+            assertEquals(1, products.size());
+            assertEquals(miniProduct.getManufacturerName(), products.firstOrNull().getManufacturer().getName());
+            assertEquals(miniProduct.getPrice(), products.firstOrNull().getPrice());
+        }
+    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java
index 8d19eac2a..c5524dfee 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/UniversalTextTokenizerTest.java
@@ -20,12 +20,13 @@
 import org.dizitart.no2.Nitrite;
 import org.dizitart.no2.NitriteBuilder;
 import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.index.IndexType;
 import org.dizitart.no2.index.NitriteTextIndexer;
 import org.dizitart.no2.index.fulltext.Languages;
 import org.dizitart.no2.index.fulltext.UniversalTextTokenizer;
-import org.dizitart.no2.common.mapper.Mappable;
-import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.Cursor;
 import org.dizitart.no2.repository.ObjectRepository;
 import org.dizitart.no2.repository.annotations.Index;
@@ -34,10 +35,9 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import static org.dizitart.no2.integration.DbTestOperations.getRandomTempDbFile;
+import static org.dizitart.no2.common.module.NitriteModule.module;
 import static org.dizitart.no2.filters.Filter.ALL;
 import static org.dizitart.no2.filters.FluentFilter.where;
-import static org.dizitart.no2.common.module.NitriteModule.module;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
@@ -45,13 +45,14 @@
  * @author Anindya Chatterjee
  */
 public class UniversalTextTokenizerTest extends BaseObjectRepositoryTest {
-    private final String fileName = getRandomTempDbFile();
     private ObjectRepository textRepository;
 
     @Before
     @Override
     public void setUp() {
         openDb();
+        SimpleDocumentMapper documentMapper = (SimpleDocumentMapper) db.getConfig().nitriteMapper();
+        documentMapper.registerEntityConverter(new TextData.Converter());
 
         textRepository = db.getRepository(TextData.class);
 
@@ -157,22 +158,31 @@ public void testUniversalFullTextIndexing() {
     }
 
     @Indices(
-        @Index(value = "text", type = IndexType.FULL_TEXT)
+        @Index(fields = "text", type = IndexType.FULL_TEXT)
     )
-    public static class TextData implements Mappable {
-        public int id;
+    public static class TextData {
+        public Integer id;
         public String text;
 
-        @Override
-        public Document write(NitriteMapper mapper) {
-            return Document.createDocument("id", id)
-                .put("text", text);
-        }
+        public static class Converter implements EntityConverter {
+            @Override
+            public Class getEntityType() {
+                return TextData.class;
+            }
+
+            @Override
+            public Document toDocument(TextData entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument("id", entity.id)
+                    .put("text", entity.text);
+            }
 
-        @Override
-        public void read(NitriteMapper mapper, Document document) {
-            id = document.get("id", Integer.class);
-            text = document.get("text", String.class);
+            @Override
+            public TextData fromDocument(Document document, NitriteMapper nitriteMapper) {
+                TextData entity = new TextData();
+                entity.id = document.get("id", Integer.class);
+                entity.text = document.get("text", String.class);
+                return entity;
+            }
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Book.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Book.java
index 5a147769c..c27993fc4 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Book.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Book.java
@@ -19,7 +19,7 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.index.IndexType;
 import org.dizitart.no2.repository.annotations.Entity;
@@ -35,12 +35,12 @@
  */
 @Data
 @Entity(value = "books", indices = {
-    @Index(value = "tags", type = IndexType.NON_UNIQUE),
-    @Index(value = "description", type = IndexType.FULL_TEXT),
-    @Index(value = { "price", "publisher" })
+    @Index(fields = "tags", type = IndexType.NON_UNIQUE),
+    @Index(fields = "description", type = IndexType.FULL_TEXT),
+    @Index(fields = { "price", "publisher" })
 })
-public class Book implements Mappable {
-    @Id(fieldName = "book_id")
+public class Book {
+    @Id(fieldName = "book_id", embeddedFields = { "isbn", "book_name" })
     private BookId bookId;
 
     private String publisher;
@@ -51,22 +51,32 @@ public class Book implements Mappable {
 
     private String description;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return createDocument("book_id", mapper.convert(bookId, Document.class))
-            .put("publisher", publisher)
-            .put("price", price)
-            .put("tags", tags)
-            .put("description", description);
-    }
+    public static class BookConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return Book.class;
+        }
+
+        @Override
+        public Document toDocument(Book entity, NitriteMapper nitriteMapper) {
+            return createDocument("book_id", nitriteMapper.convert(entity.bookId, Document.class))
+                .put("publisher", entity.publisher)
+                .put("price", entity.price)
+                .put("tags", entity.tags)
+                .put("description", entity.description);
+        }
 
-    @Override
-    @SuppressWarnings("unchecked")
-    public void read(NitriteMapper mapper, Document document) {
-        bookId = mapper.convert(document.get("book_id"), BookId.class);
-        publisher = document.get("publisher", String.class);
-        price = document.get("price", Double.class);
-        tags = (List) document.get("tags", List.class);
-        description = document.get("description", String.class);
+        @Override
+        @SuppressWarnings("unchecked")
+        public Book fromDocument(Document document, NitriteMapper nitriteMapper) {
+            Book entity = new Book();
+            entity.bookId = nitriteMapper.convert(document.get("book_id"), BookId.class);
+            entity.publisher = document.get("publisher", String.class);
+            entity.price = document.get("price", Double.class);
+            entity.tags = (List) document.get("tags", List.class);
+            entity.description = document.get("description", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java
index 239b493d3..4aa11f1af 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/BookId.java
@@ -19,9 +19,8 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
-import org.dizitart.no2.repository.annotations.Embedded;
 
 import static org.dizitart.no2.collection.Document.createDocument;
 
@@ -29,26 +28,34 @@
  * @author Anindya Chatterjee
  */
 @Data
-public class BookId implements Mappable {
-    @Embedded(order = 0)
+public class BookId {
     private String isbn;
 
-    @Embedded(order = 1, fieldName = "book_name")
     private String name;
 
     private String author;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return createDocument("isbn", isbn)
-            .put("book_name", name)
-            .put("author", author);
-    }
-
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        isbn = document.get("isbn", String.class);
-        name = document.get("book_name", String.class);
-        author = document.get("author", String.class);
+    public static class BookIdConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return BookId.class;
+        }
+
+        @Override
+        public Document toDocument(BookId entity, NitriteMapper nitriteMapper) {
+            return createDocument("isbn", entity.isbn)
+                .put("book_name", entity.name)
+                .put("author", entity.author);
+        }
+
+        @Override
+        public BookId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            BookId entity = new BookId();
+            entity.isbn = document.get("isbn", String.class);
+            entity.name = document.get("book_name", String.class);
+            entity.author = document.get("author", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java
index d3c2ba644..6f0470d26 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ChildClass.java
@@ -20,9 +20,12 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.InheritIndices;
 
+import java.util.Date;
+
 /**
  * @author Anindya Chatterjee
  */
@@ -32,14 +35,30 @@
 public class ChildClass extends ParentClass {
     private String name;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return super.write(mapper).put("name", name);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return ChildClass.class;
+        }
+
+        @Override
+        public Document toDocument(ChildClass entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("name", entity.getName())
+                .put("id", entity.getId())
+                .put("date", entity.getDate())
+                .put("text", entity.getText());
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        super.read(mapper, document);
-        name = document.get("name", String.class);
+        @Override
+        public ChildClass fromDocument(Document document, NitriteMapper nitriteMapper) {
+            ChildClass entity = new ChildClass();
+            entity.setId(document.get("id", Long.class));
+            entity.setDate(document.get("date", Date.class));
+            entity.setText(document.get("text", String.class));
+            entity.setName(document.get("name", String.class));
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java
index a8161c4e2..4e736981d 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassA.java
@@ -22,14 +22,14 @@
 import lombok.Setter;
 import lombok.ToString;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 import java.util.UUID;
 
 @EqualsAndHashCode
 @ToString
-public class ClassA implements Mappable {
+public class ClassA {
     @Getter
     @Setter
     private ClassB b;
@@ -53,23 +53,33 @@ public static ClassA create(int seed) {
         return classA;
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("b", b != null ? b.write(mapper) : null)
-            .put("uid", uid)
-            .put("string", string)
-            .put("blob", blob);
-    }
+    public static class ClassAConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return ClassA.class;
+        }
+
+        @Override
+        public Document toDocument(ClassA entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("b", nitriteMapper.convert(entity.b, Document.class))
+                .put("uid", entity.uid)
+                .put("string", entity.string)
+                .put("blob", entity.blob);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        if (document.get("b") != null) {
-            b = new ClassB();
-            b.read(mapper, document.get("b", Document.class));
+        @Override
+        public ClassA fromDocument(Document document, NitriteMapper nitriteMapper) {
+            ClassA entity = new ClassA();
+            if (document.get("b") != null) {
+                Document doc = document.get("b", Document.class);
+                entity.b = nitriteMapper.convert(doc, ClassB.class);
+            }
+            entity.uid = document.get("uid", UUID.class);
+            entity.string = document.get("string", String.class);
+            entity.blob = document.get("blob", byte[].class);
+            return entity;
         }
-        uid = document.get("uid", UUID.class);
-        string = document.get("string", String.class);
-        blob = document.get("blob", byte[].class);
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java
index a546b91e2..db398e31a 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassB.java
@@ -21,16 +21,14 @@
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
-import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
-import org.dizitart.no2.common.mapper.NitriteMapper;
 
 @EqualsAndHashCode
 @ToString
-class ClassB implements Comparable, Mappable {
+class ClassB implements Comparable {
     @Getter
     @Setter
     private int number;
+
     @Getter
     @Setter
     private String text;
@@ -47,16 +45,4 @@ public int compareTo(ClassB o) {
         return Integer.compare(number, o.number);
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("number", number)
-            .put("text", text);
-    }
-
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        number = document.get("number", Integer.class);
-        text = document.get("text", String.class);
-    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java
new file mode 100644
index 000000000..6d5f88bab
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassBConverter.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.data;
+
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+
+public class ClassBConverter implements EntityConverter {
+
+    @Override
+    public Class getEntityType() {
+        return ClassB.class;
+    }
+
+    @Override
+    public Document toDocument(ClassB entity, NitriteMapper nitriteMapper) {
+        return Document.createDocument()
+            .put("number", entity.getNumber())
+            .put("text", entity.getText());
+    }
+
+    @Override
+    public ClassB fromDocument(Document document, NitriteMapper nitriteMapper) {
+        ClassB entity = new ClassB();
+        if (document.get("number") != null) {
+            entity.setNumber(document.get("number", Integer.class));
+        }
+        entity.setText(document.get("text", String.class));
+        return entity;
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java
index 860fe1e93..0dd8d58d6 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ClassC.java
@@ -22,12 +22,12 @@
 import lombok.Setter;
 import lombok.ToString;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 @EqualsAndHashCode
 @ToString
-public class ClassC implements Mappable {
+public class ClassC {
     @Getter
     @Setter
     private long id;
@@ -46,21 +46,37 @@ public static ClassC create(int seed) {
         return classC;
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("id", id)
-            .put("digit", digit)
-            .put("parent", parent != null ? parent.write(mapper) : null);
-    }
+    public static class ClassCConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return ClassC.class;
+        }
+
+        @Override
+        public Document toDocument(ClassC entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("id", entity.id)
+                .put("digit", entity.digit)
+                .put("parent", nitriteMapper.convert(entity.parent, Document.class));
+        }
+
+        @Override
+        public ClassC fromDocument(Document document, NitriteMapper nitriteMapper) {
+            ClassC entity = new ClassC();
+            if (document.get("id") != null) {
+                entity.id = document.get("id", Long.class);
+            }
+
+            if (document.get("digit") != null) {
+                entity.digit = document.get("digit", Double.class);
+            }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        id = document.get("id", Long.class);
-        digit = document.get("digit", Double.class);
-        if (document.get("parent") != null) {
-            parent = new ClassA();
-            parent.read(mapper, document.get("parent", Document.class));
+            if (document.get("parent") != null) {
+                Document doc = document.get("parent", Document.class);
+                entity.parent = nitriteMapper.convert(doc, ClassA.class);
+            }
+            return entity;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Company.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Company.java
index 944e17358..649053261 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Company.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Company.java
@@ -21,7 +21,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 import org.dizitart.no2.repository.annotations.Index;
@@ -37,9 +37,9 @@
  */
 @EqualsAndHashCode
 @Indices({
-    @Index(value = "companyName")
+    @Index(fields = "companyName")
 })
-public class Company implements Serializable, Mappable {
+public class Company implements Serializable {
     @Id(fieldName = "company_id")
     @Getter
     @Setter
@@ -61,25 +61,6 @@ public class Company implements Serializable, Mappable {
     @Setter
     private Map> employeeRecord;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("company_id", companyId)
-            .put("companyName", companyName)
-            .put("dateCreated", dateCreated)
-            .put("departments", departments)
-            .put("employeeRecord", employeeRecord);
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public void read(NitriteMapper mapper, Document document) {
-        companyId = document.get("company_id", Long.class);
-        companyName = document.get("companyName", String.class);
-        dateCreated = document.get("dateCreated", Date.class);
-        departments = document.get("departments", List.class);
-        employeeRecord = document.get("employeeRecord", Map.class);
-    }
-
     @Override
     public String toString() {
         return "Company{" +
@@ -89,4 +70,32 @@ public String toString() {
             ", departments=" + departments +
             '}';
     }
+
+    public static class CompanyConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return Company.class;
+        }
+
+        @Override
+        public Document toDocument(Company entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("company_id", entity.companyId)
+                .put("companyName", entity.companyName)
+                .put("dateCreated", entity.dateCreated)
+                .put("departments", entity.departments)
+                .put("employeeRecord", entity.employeeRecord);
+        }
+
+        @Override
+        public Company fromDocument(Document document, NitriteMapper nitriteMapper) {
+            Company entity = new Company();
+            entity.companyId = document.get("company_id", Long.class);
+            entity.companyName = document.get("companyName", String.class);
+            entity.dateCreated = document.get("dateCreated", Date.class);
+            entity.departments = document.get("departments", List.class);
+            entity.employeeRecord = document.get("employeeRecord", Map.class);
+            return entity;
+        }
+    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java
index 26e0dcf4a..6d83c2940 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/DataGenerator.java
@@ -19,6 +19,9 @@
 
 import com.github.javafaker.Faker;
 import lombok.val;
+import org.dizitart.no2.integration.repository.decorator.Manufacturer;
+import org.dizitart.no2.integration.repository.decorator.Product;
+import org.dizitart.no2.integration.repository.decorator.ProductId;
 
 import java.nio.charset.StandardCharsets;
 import java.util.*;
@@ -102,6 +105,15 @@ public static Book randomBook() {
         return book;
     }
 
+    public static Product randomProduct() {
+        Product product = new Product();
+        product.setProductName(faker.name().name());
+        product.setProductId(randomProductId());
+        product.setManufacturer(randomManufacturer());
+        product.setPrice(Double.parseDouble(faker.commerce().price()));
+        return product;
+    }
+
     private static List departments() {
         return new ArrayList() {{
             add("dev");
@@ -114,4 +126,19 @@ private static List departments() {
             add("support");
         }};
     }
+
+    private static ProductId randomProductId() {
+        ProductId productId = new ProductId();
+        productId.setProductCode(faker.code().ean13());
+        productId.setUniqueId(UUID.randomUUID().toString());
+        return productId;
+    }
+
+    private static Manufacturer randomManufacturer() {
+        Manufacturer manufacturer = new Manufacturer();
+        manufacturer.setUniqueId(random.nextInt());
+        manufacturer.setName(faker.name().name());
+        manufacturer.setAddress(faker.address().fullAddress());
+        return manufacturer;
+    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java
index cb5304e29..92c710d2e 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ElemMatch.java
@@ -19,7 +19,7 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 import java.util.ArrayList;
@@ -29,38 +29,46 @@
  * @author Anindya Chatterjee
  */
 @Data
-public class ElemMatch implements Mappable {
-    private long id;
+public class ElemMatch {
+    private Long id;
     private String[] strArray;
     private ProductScore[] productScores;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        List list = new ArrayList<>();
-        if (productScores != null) {
-            for (ProductScore productScore : productScores) {
-                Document document = productScore.write(mapper);
-                list.add(document);
-            }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return ElemMatch.class;
         }
 
-        return Document.createDocument("id", id)
-            .put("strArray", strArray)
-            .put("productScores", list);
-    }
+        @Override
+        public Document toDocument(ElemMatch entity, NitriteMapper nitriteMapper) {
+            List list = new ArrayList<>();
+            if (entity.productScores != null) {
+                for (ProductScore productScore : entity.productScores) {
+                    Document document = nitriteMapper.convert(productScore, Document.class);
+                    list.add(document);
+                }
+            }
+
+            return Document.createDocument("id", entity.id)
+                .put("strArray", entity.strArray)
+                .put("productScores", list);
+        }
 
-    @Override
-    @SuppressWarnings("unchecked")
-    public void read(NitriteMapper mapper, Document document) {
-        id = document.get("id", Long.class);
-        strArray = document.get("strArray", String[].class);
-        List list = document.get("productScores", List.class);
-        if (list != null) {
-            productScores = new ProductScore[list.size()];
-            for (int i = 0; i < list.size(); i++) {
-                productScores[i] = new ProductScore();
-                productScores[i].read(mapper, list.get(i));
+        @Override
+        public ElemMatch fromDocument(Document document, NitriteMapper nitriteMapper) {
+            ElemMatch entity = new ElemMatch();
+            entity.id = document.get("id", Long.class);
+            entity.strArray = document.get("strArray", String[].class);
+            List list = document.get("productScores", List.class);
+            if (list != null) {
+                entity.productScores = new ProductScore[list.size()];
+                for (int i = 0; i < list.size(); i++) {
+                    entity.productScores[i] = nitriteMapper.convert(list.get(i), ProductScore.class);
+                }
             }
+            return entity;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java
index 7204f5654..f89b6ecc9 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Employee.java
@@ -22,9 +22,9 @@
 import lombok.Setter;
 import lombok.ToString;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.index.IndexType;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.index.IndexType;
 import org.dizitart.no2.repository.annotations.Id;
 import org.dizitart.no2.repository.annotations.Index;
 
@@ -36,10 +36,10 @@
  */
 @ToString
 @EqualsAndHashCode
-@Index(value = "joinDate", type = IndexType.NON_UNIQUE)
-@Index(value = "address", type = IndexType.FULL_TEXT)
-@Index(value = "employeeNote.text", type = IndexType.FULL_TEXT)
-public class Employee implements Serializable, Mappable {
+@Index(fields = "joinDate", type = IndexType.NON_UNIQUE)
+@Index(fields = "address", type = IndexType.FULL_TEXT)
+@Index(fields = "employeeNote.text", type = IndexType.FULL_TEXT)
+public class Employee implements Serializable {
     @Id
     @Getter
     @Setter
@@ -82,28 +82,39 @@ public Employee(Employee copy) {
         emailAddress = copy.emailAddress;
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("empId", empId)
-            .put("joinDate", joinDate)
-            .put("address", address)
-            .put("blob", blob)
-            .put("emailAddress", emailAddress)
-            .put("employeeNote", employeeNote != null ? employeeNote.write(mapper) : null);
-    }
+    public static class EmployeeConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return Employee.class;
+        }
+
+        @Override
+        public Document toDocument(Employee entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("empId", entity.empId)
+                .put("joinDate", entity.joinDate)
+                .put("address", entity.address)
+                .put("blob", entity.blob)
+                .put("emailAddress", entity.emailAddress)
+                .put("employeeNote", nitriteMapper.convert(entity.employeeNote, Document.class));
+        }
+
+        @Override
+        public Employee fromDocument(Document document, NitriteMapper nitriteMapper) {
+            Employee entity = new Employee();
+
+            entity.empId = document.get("empId", Long.class);
+            entity.joinDate = document.get("joinDate", Date.class);
+            entity.address = document.get("address", String.class);
+            entity.blob = document.get("blob", byte[].class);
+            entity.emailAddress = document.get("emailAddress", String.class);
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        empId = document.get("empId", Long.class);
-        joinDate = document.get("joinDate", Date.class);
-        address = document.get("address", String.class);
-        blob = document.get("blob", byte[].class);
-        emailAddress = document.get("emailAddress", String.class);
-
-        if (document.get("employeeNote") != null) {
-            employeeNote = new Note();
-            employeeNote.read(mapper, document.get("employeeNote", Document.class));
+            if (document.get("employeeNote") != null) {
+                Document doc = document.get("employeeNote", Document.class);
+                entity.employeeNote = nitriteMapper.convert(doc, Note.class);;
+            }
+            return entity;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java
new file mode 100644
index 000000000..9bf6bcbf6
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/EmptyClass.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.data;
+
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+
+public class EmptyClass {
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return EmptyClass.class;
+        }
+
+        @Override
+        public Document toDocument(EmptyClass entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument();
+        }
+
+        @Override
+        public EmptyClass fromDocument(Document document, NitriteMapper nitriteMapper) {
+            return new EmptyClass();
+        }
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java
index 8b9d4b23b..7003db015 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/EncryptedPerson.java
@@ -19,7 +19,7 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Entity;
 
@@ -30,25 +30,35 @@
  */
 @Data
 @Entity
-public class EncryptedPerson implements Mappable {
+public class EncryptedPerson {
     private String name;
     private String creditCardNumber;
     private String cvv;
     private Date expiryDate;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("name", name)
-            .put("creditCardNumber", creditCardNumber)
-            .put("cvv", cvv)
-            .put("expiryDate", expiryDate);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return EncryptedPerson.class;
+        }
+
+        @Override
+        public Document toDocument(EncryptedPerson entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("name", entity.name)
+                .put("creditCardNumber", entity.creditCardNumber)
+                .put("cvv", entity.cvv)
+                .put("expiryDate", entity.expiryDate);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        creditCardNumber = document.get("creditCardNumber", String.class);
-        cvv = document.get("cvv", String.class);
-        expiryDate = document.get("expiryDate", Date.class);
+        @Override
+        public EncryptedPerson fromDocument(Document document, NitriteMapper nitriteMapper) {
+            EncryptedPerson entity = new EncryptedPerson();
+            entity.name = document.get("name", String.class);
+            entity.creditCardNumber = document.get("creditCardNumber", String.class);
+            entity.cvv = document.get("cvv", String.class);
+            entity.expiryDate = document.get("expiryDate", Date.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Note.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Note.java
index 67dc307d1..7bf7f1726 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Note.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/Note.java
@@ -21,7 +21,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 import java.io.Serializable;
@@ -30,7 +30,7 @@
  * @author Anindya Chatterjee.
  */
 @EqualsAndHashCode
-public class Note implements Serializable, Mappable {
+public class Note implements Serializable {
     @Getter
     @Setter
     private Long noteId;
@@ -38,14 +38,26 @@ public class Note implements Serializable, Mappable {
     @Setter
     private String text;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument().put("noteId", noteId).put("text", text);
-    }
+    public static class NoteConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return Note.class;
+        }
+
+        @Override
+        public Document toDocument(Note entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("noteId", entity.noteId)
+                .put("text", entity.text);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        noteId = document.get("noteId", Long.class);
-        text = document.get("text", String.class);
+        @Override
+        public Note fromDocument(Document document, NitriteMapper nitriteMapper) {
+            Note entity = new Note();
+            entity.noteId = document.get("noteId", Long.class);
+            entity.text = document.get("text", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java
index bc2514543..f27a3f869 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ParentClass.java
@@ -19,8 +19,6 @@
 
 import lombok.Getter;
 import lombok.Setter;
-import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 import org.dizitart.no2.repository.annotations.Index;
 
@@ -31,23 +29,9 @@
  */
 @Getter
 @Setter
-@Index(value = "date")
+@Index(fields = "date")
 public class ParentClass extends SuperDuperClass {
     @Id
     protected Long id;
     private Date date;
-
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return super.write(mapper)
-            .put("id", id)
-            .put("date", date);
-    }
-
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        super.read(mapper, document);
-        id = document.get("id", Long.class);
-        date = document.get("date", Date.class);
-    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java
index 2b90075fa..b6dcd03a3 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/PersonEntity.java
@@ -19,9 +19,9 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.index.IndexType;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.index.IndexType;
 import org.dizitart.no2.repository.annotations.Entity;
 import org.dizitart.no2.repository.annotations.Id;
 import org.dizitart.no2.repository.annotations.Index;
@@ -34,10 +34,10 @@
  */
 @Data
 @Entity(value = "MyPerson", indices = {
-    @Index(value = "name", type = IndexType.FULL_TEXT),
-    @Index(value = "status", type = IndexType.NON_UNIQUE)
+    @Index(fields = "name", type = IndexType.FULL_TEXT),
+    @Index(fields = "status", type = IndexType.NON_UNIQUE)
 })
-public class PersonEntity implements Mappable {
+public class PersonEntity {
     @Id
     private String uuid;
     private String name;
@@ -56,24 +56,34 @@ public PersonEntity(String name) {
         this.dateCreated = new Date();
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("uuid", uuid)
-            .put("name", name)
-            .put("status", status)
-            .put("friend", friend != null ? friend.write(mapper) : null)
-            .put("dateCreated", dateCreated);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return PersonEntity.class;
+        }
+
+        @Override
+        public Document toDocument(PersonEntity entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("uuid", entity.uuid)
+                .put("name", entity.name)
+                .put("status", entity.status)
+                .put("friend", entity.friend != null ? nitriteMapper.convert(entity.friend, Document.class) : null)
+                .put("dateCreated", entity.dateCreated);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        if (document != null) {
-            uuid = document.get("uuid", String.class);
-            name = document.get("name", String.class);
-            status = document.get("status", String.class);
-            dateCreated = document.get("dateCreated", Date.class);
-            friend = new PersonEntity();
-            friend.read(mapper, document.get("friend", Document.class));
+        @Override
+        public PersonEntity fromDocument(Document document, NitriteMapper nitriteMapper) {
+            if (document != null) {
+                PersonEntity entity = new PersonEntity();
+                entity.uuid = document.get("uuid", String.class);
+                entity.name = document.get("name", String.class);
+                entity.status = document.get("status", String.class);
+                entity.dateCreated = document.get("dateCreated", Date.class);
+                entity.friend = nitriteMapper.convert(document.get("friend", Document.class), PersonEntity.class);
+                return entity;
+            }
+            return null;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java
index 65550d182..23c9d16ff 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/ProductScore.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 /**
@@ -28,9 +28,9 @@
  */
 @Getter
 @Setter
-public class ProductScore implements Mappable {
+public class ProductScore {
     private String product;
-    private int score;
+    private Integer score;
 
     public ProductScore() {
     }
@@ -40,15 +40,25 @@ public ProductScore(String product, int score) {
         this.score = score;
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("product", product)
-            .put("score", score);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return ProductScore.class;
+        }
+
+        @Override
+        public Document toDocument(ProductScore entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("product", entity.product)
+                .put("score", entity.score);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        product = document.get("product", String.class);
-        score = document.get("score", Integer.class);
+        @Override
+        public ProductScore fromDocument(Document document, NitriteMapper nitriteMapper) {
+            ProductScore entity = new ProductScore();
+            entity.product = document.get("product", String.class);
+            entity.score = document.get("score", Integer.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java
index 1ecadb243..5d9208b86 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/RepeatableIndexTest.java
@@ -19,34 +19,44 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.index.IndexType;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
+import org.dizitart.no2.index.IndexType;
 import org.dizitart.no2.repository.annotations.Index;
 
 /**
  * @author Anindya Chatterjee
  */
 @Data
-@Index(value = "firstName")
-@Index(value = "age", type = IndexType.NON_UNIQUE)
-@Index(value = "lastName", type = IndexType.FULL_TEXT)
-public class RepeatableIndexTest implements Mappable {
+@Index(fields = "firstName")
+@Index(fields = "age", type = IndexType.NON_UNIQUE)
+@Index(fields = "lastName", type = IndexType.FULL_TEXT)
+public class RepeatableIndexTest {
     private String firstName;
     private Integer age;
     private String lastName;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("firstName", firstName)
-            .put("age", age)
-            .put("lastName", lastName);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return RepeatableIndexTest.class;
+        }
+
+        @Override
+        public Document toDocument(RepeatableIndexTest entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("firstName", entity.firstName)
+                .put("age", entity.age)
+                .put("lastName", entity.lastName);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        firstName = document.get("firstName", String.class);
-        age = document.get("age", Integer.class);
-        lastName = document.get("lastName", String.class);
+        @Override
+        public RepeatableIndexTest fromDocument(Document document, NitriteMapper nitriteMapper) {
+            RepeatableIndexTest entity = new RepeatableIndexTest();
+            entity.firstName = document.get("firstName", String.class);
+            entity.age = document.get("age", Integer.class);
+            entity.lastName = document.get("lastName", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java
index d1cb199c4..33dc06073 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/StressRecord.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 /**
@@ -28,28 +28,37 @@
  */
 @Getter
 @Setter
-public class StressRecord implements Mappable {
+public class StressRecord {
     private String firstName;
-    private boolean processed;
+    private Boolean processed;
     private String lastName;
-    private boolean failed;
+    private Boolean failed;
     private String notes;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument().put("firstName", firstName)
-            .put("processed", processed)
-            .put("lastName", lastName)
-            .put("failed", failed)
-            .put("notes", notes);
-    }
+    public static class Converter implements EntityConverter {
+        @Override
+        public Class getEntityType() {
+            return StressRecord.class;
+        }
+
+        @Override
+        public Document toDocument(StressRecord entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument().put("firstName", entity.firstName)
+                .put("processed", entity.processed)
+                .put("lastName", entity.lastName)
+                .put("failed", entity.failed)
+                .put("notes", entity.notes);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        firstName = document.get("firstName", String.class);
-        processed = document.get("processed", Boolean.class);
-        lastName = document.get("lastName", String.class);
-        failed = document.get("failed", Boolean.class);
-        notes = document.get("notes", String.class);
+        @Override
+        public StressRecord fromDocument(Document document, NitriteMapper nitriteMapper) {
+            StressRecord entity = new StressRecord();
+            entity.firstName = document.get("firstName", String.class);
+            entity.processed = document.get("processed", Boolean.class);
+            entity.lastName = document.get("lastName", String.class);
+            entity.failed = document.get("failed", Boolean.class);
+            entity.notes = document.get("notes", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java
index 8eaa3faf1..ddea32812 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SubEmployee.java
@@ -21,7 +21,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 import java.util.Date;
@@ -30,7 +30,7 @@
  * @author Anindya Chatterjee.
  */
 @EqualsAndHashCode
-public class SubEmployee implements Mappable {
+public class SubEmployee {
     @Getter
     @Setter
     private Long empId;
@@ -43,18 +43,28 @@ public class SubEmployee implements Mappable {
     @Setter
     private String address;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("empId", empId)
-            .put("joinDate", joinDate)
-            .put("address", address);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return SubEmployee.class;
+        }
+
+        @Override
+        public Document toDocument(SubEmployee entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("empId", entity.empId)
+                .put("joinDate", entity.joinDate)
+                .put("address", entity.address);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        empId = document.get("empId", Long.class);
-        joinDate = document.get("joinDate", Date.class);
-        address = document.get("address", String.class);
+        @Override
+        public SubEmployee fromDocument(Document document, NitriteMapper nitriteMapper) {
+            SubEmployee entity = new SubEmployee();
+            entity.empId = document.get("empId", Long.class);
+            entity.joinDate = document.get("joinDate", Date.class);
+            entity.address = document.get("address", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java
index 198f16d0c..523cebb87 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/SuperDuperClass.java
@@ -19,10 +19,7 @@
 
 import lombok.Getter;
 import lombok.Setter;
-import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.index.IndexType;
-import org.dizitart.no2.common.mapper.Mappable;
-import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Index;
 
 /**
@@ -30,17 +27,7 @@
  */
 @Getter
 @Setter
-@Index(value = "text", type = IndexType.FULL_TEXT)
-public class SuperDuperClass implements Mappable {
+@Index(fields = "text", type = IndexType.FULL_TEXT)
+public class SuperDuperClass {
     private String text;
-
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("text", text);
-    }
-
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        text = document.get("text", String.class);
-    }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java
index 79cafe0dc..bdeef8744 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithClassField.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -29,20 +29,29 @@
  */
 @Getter
 @Setter
-public class WithClassField implements Mappable {
+public class WithClassField {
     @Id
     private String name;
     private Class clazz;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("name", name)
-            .put("clazz", clazz);
-    }
+    public static class Converter implements EntityConverter {
+        @Override
+        public Class getEntityType() {
+            return WithClassField.class;
+        }
+
+        @Override
+        public Document toDocument(WithClassField entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("name", entity.name)
+                .put("clazz", entity.clazz);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        clazz = document.get("clazz", Class.class);
+        @Override
+        public WithClassField fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithClassField entity = new WithClassField();
+            entity.name = document.get("name", String.class);
+            entity.clazz = document.get("clazz", Class.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java
index 229f1aaf7..125e94b05 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithDateId.java
@@ -21,7 +21,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 import java.util.Date;
@@ -32,19 +32,29 @@
 @Getter
 @Setter
 @EqualsAndHashCode
-public class WithDateId implements Mappable {
+public class WithDateId {
     private Date id;
     private String name;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("name", name)
-            .put("id", id);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithDateId.class;
+        }
+
+        @Override
+        public Document toDocument(WithDateId entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("name", entity.name)
+                .put("id", entity.id);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        id = document.get("id", Date.class);
+        @Override
+        public WithDateId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithDateId entity = new WithDateId();
+            entity.name = document.get("name", String.class);
+            entity.id = document.get("id", Date.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java
index a0048a6bb..15792f12e 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithEmptyStringId.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -29,17 +29,27 @@
  */
 @Getter
 @Setter
-public class WithEmptyStringId implements Mappable {
+public class WithEmptyStringId {
     @Id
     private String name;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("name", name);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithEmptyStringId.class;
+        }
+
+        @Override
+        public Document toDocument(WithEmptyStringId entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("name", entity.name);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
+        @Override
+        public WithEmptyStringId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithEmptyStringId entity = new WithEmptyStringId();
+            entity.name = document.get("name", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java
index 2364be76d..6748256c6 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNitriteId.java
@@ -20,7 +20,7 @@
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteId;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -28,21 +28,31 @@
  * @author Anindya Chatterjee
  */
 @Data
-public class WithNitriteId implements Mappable {
+public class WithNitriteId {
     @Id
     public NitriteId idField;
     public String name;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("idField", idField)
-            .put("name", name);
-    }
+    public static class WithNitriteIdConverter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithNitriteId.class;
+        }
+
+        @Override
+        public Document toDocument(WithNitriteId entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("idField", entity.idField)
+                .put("name", entity.name);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        idField = document.get("idField", NitriteId.class);
-        name = document.get("name", String.class);
+        @Override
+        public WithNitriteId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithNitriteId entity = new WithNitriteId();
+            entity.idField = document.get("idField", NitriteId.class);
+            entity.name = document.get("name", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java
index 0dc3fb68b..dfac35bfa 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithNullId.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -29,21 +29,31 @@
  */
 @Getter
 @Setter
-public class WithNullId implements Mappable {
+public class WithNullId {
     @Id
     private String name;
-    private long number;
+    private Long number;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("name", name)
-            .put("number", number);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithNullId.class;
+        }
+
+        @Override
+        public Document toDocument(WithNullId entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("name", entity.name)
+                .put("number", entity.number);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        number = document.get("number", Long.class);
+        @Override
+        public WithNullId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithNullId entity = new WithNullId();
+            entity.name = document.get("name", String.class);
+            entity.number = document.get("number", Long.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java
index 5f00f97d4..634a90fb8 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithObjectId.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -29,17 +29,27 @@
  */
 @Getter
 @Setter
-public class WithObjectId implements Mappable {
+public class WithObjectId {
     @Id
     private WithOutId withOutId;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("withOutId", withOutId);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithObjectId.class;
+        }
+
+        @Override
+        public Document toDocument(WithObjectId entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("withOutId", entity.withOutId);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        withOutId = document.get("withOutId", WithOutId.class);
+        @Override
+        public WithObjectId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithObjectId entity = new WithObjectId();
+            entity.withOutId = document.get("withOutId", WithOutId.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java
index a7a2c56c9..e6783a89e 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutGetterSetter.java
@@ -19,32 +19,41 @@
 
 import lombok.EqualsAndHashCode;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 /**
  * @author Anindya Chatterjee.
  */
 @EqualsAndHashCode
-public class WithOutGetterSetter implements Mappable {
+public class WithOutGetterSetter {
     private String name;
-    private long number;
+    private Long number;
 
     public WithOutGetterSetter() {
         name = "test";
-        number = 2;
+        number = 2L;
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("name", name)
-            .put("number", number);
+    public static class Converter implements EntityConverter {
 
-    }
+        @Override
+        public Class getEntityType() {
+            return WithOutGetterSetter.class;
+        }
+
+        @Override
+        public Document toDocument(WithOutGetterSetter entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("name", entity.name)
+                .put("number", entity.number);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        number = document.get("number", Long.class);
+        @Override
+        public WithOutGetterSetter fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithOutGetterSetter entity = new WithOutGetterSetter();
+            entity.name = document.get("name", String.class);
+            entity.number = document.get("number", Long.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java
index 09b63c2c2..4685e7726 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithOutId.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 /**
@@ -28,25 +28,35 @@
  */
 @Getter
 @Setter
-public class WithOutId implements Comparable, Mappable {
+public class WithOutId implements Comparable {
     private String name;
-    private long number;
+    private Long number;
 
     @Override
     public int compareTo(WithOutId o) {
         return Long.compare(number, o.number);
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("name", name)
-            .put("number", number);
-    }
+    public static class Converter implements EntityConverter {
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        number = document.get("number", Long.class);
+        @Override
+        public Class getEntityType() {
+            return WithOutId.class;
+        }
+
+        @Override
+        public Document toDocument(WithOutId entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("name", entity.name)
+                .put("number", entity.number);
+        }
+
+        @Override
+        public WithOutId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithOutId entity = new WithOutId();
+            entity.name = document.get("name", String.class);
+            entity.number = document.get("number", Long.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java
index a425209fb..de08cf19b 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPrivateConstructor.java
@@ -19,38 +19,47 @@
 
 import lombok.EqualsAndHashCode;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 
 /**
  * @author Anindya Chatterjee.
  */
 @EqualsAndHashCode
-public class WithPrivateConstructor implements Mappable {
+public class WithPrivateConstructor {
     private String name;
-    private long number;
+    private Long number;
 
     private WithPrivateConstructor() {
         name = "test";
-        number = 2;
+        number = 2L;
     }
 
-    public static WithPrivateConstructor create(final String name, final long number) {
+    public static WithPrivateConstructor create(final String name, final Long number) {
         WithPrivateConstructor obj = new WithPrivateConstructor();
         obj.number = number;
         obj.name = name;
         return obj;
     }
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("name", name)
-            .put("number", number);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithPrivateConstructor.class;
+        }
+
+        @Override
+        public Document toDocument(WithPrivateConstructor entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("name", entity.name)
+                .put("number", entity.number);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        number = document.get("number", Long.class);
+        @Override
+        public WithPrivateConstructor fromDocument(Document document, NitriteMapper nitriteMapper) {
+            String name = document.get("name", String.class);
+            Long number = document.get("number", Long.class);
+            return WithPrivateConstructor.create(name, number);
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java
index 793388231..81b69a0c1 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithPublicField.java
@@ -18,27 +18,37 @@
 package org.dizitart.no2.integration.repository.data;
 
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
 /**
  * @author Anindya Chatterjee.
  */
-public class WithPublicField implements Mappable {
+public class WithPublicField {
     @Id
     public String name;
-    public long number;
+    public Long number;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("name", name)
-            .put("number", number);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithPublicField.class;
+        }
+
+        @Override
+        public Document toDocument(WithPublicField entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("name", entity.name)
+                .put("number", entity.number);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        name = document.get("name", String.class);
-        number = document.get("number", Long.class);
+        @Override
+        public WithPublicField fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithPublicField entity = new WithPublicField();
+            entity.name = document.get("name", String.class);
+            entity.number = document.get("number", Long.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java
index 9c1ade657..76ea86243 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithTransientField.java
@@ -20,7 +20,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -29,19 +29,29 @@
  */
 @Getter
 @Setter
-public class WithTransientField implements Mappable {
+public class WithTransientField {
     private transient String name;
     @Id
-    private long number;
+    private Long number;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("number", number);
-    }
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return WithTransientField.class;
+        }
+
+        @Override
+        public Document toDocument(WithTransientField entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("number", entity.number);
+        }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        number = document.get("number", Long.class);
+        @Override
+        public WithTransientField fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithTransientField entity = new WithTransientField();
+            entity.number = document.get("number", Long.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java
index 9ecb9a68b..9c721c80c 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/data/WithoutEmbeddedId.java
@@ -19,7 +19,7 @@
 
 import lombok.Data;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -27,39 +27,60 @@
  * @author Anindya Chatterjee
  */
 @Data
-public class WithoutEmbeddedId implements Mappable {
+public class WithoutEmbeddedId {
     @Id
     private NestedId nestedId;
     private String data;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument()
-            .put("nestedId", nestedId.write(mapper))
-            .put("data", data);
-    }
+    @Data
+    public static class NestedId {
+        private Long id;
+
+        public static class Converter implements EntityConverter {
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        Document nestedId = document.get("nestedId", Document.class);
-        this.nestedId = mapper.convert(nestedId, NestedId.class);
-        this.data = document.get("data", String.class);
+            @Override
+            public Class getEntityType() {
+                return NestedId.class;
+            }
+
+            @Override
+            public Document toDocument(NestedId entity, NitriteMapper nitriteMapper) {
+                return Document.createDocument()
+                    .put("id", entity.id);
+            }
+
+            @Override
+            public NestedId fromDocument(Document document, NitriteMapper nitriteMapper) {
+                NestedId entity = new NestedId();
+                entity.id = document.get("id", Long.class);
+                return entity;
+            }
+        }
     }
 
+    public static class Converter implements EntityConverter {
 
-    @Data
-    public static class NestedId implements Mappable {
-        private Long id;
+        @Override
+        public Class getEntityType() {
+            return WithoutEmbeddedId.class;
+        }
 
         @Override
-        public Document write(NitriteMapper mapper) {
+        public Document toDocument(WithoutEmbeddedId entity, NitriteMapper nitriteMapper) {
             return Document.createDocument()
-                .put("id", id);
+                .put("nestedId", nitriteMapper.convert(entity.nestedId, Document.class))
+                .put("data", entity.data);
         }
 
         @Override
-        public void read(NitriteMapper mapper, Document document) {
-            id = document.get("id", Long.class);
+        public WithoutEmbeddedId fromDocument(Document document, NitriteMapper nitriteMapper) {
+            WithoutEmbeddedId entity = new WithoutEmbeddedId();
+            Document nestedId = document.get("nestedId", Document.class);
+
+            entity.nestedId = nitriteMapper.convert(nestedId, NestedId.class);
+            entity.data = document.get("data", String.class);
+
+            return entity;
         }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java
new file mode 100644
index 000000000..c6bfe8bb9
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/Manufacturer.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import lombok.Data;
+
+@Data
+public class Manufacturer {
+    private String name;
+    private String address;
+    private Integer uniqueId;
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java
new file mode 100644
index 000000000..f31d7b0bb
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerConverter.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+
+public class ManufacturerConverter implements EntityConverter {
+    @Override
+    public Class getEntityType() {
+        return Manufacturer.class;
+    }
+
+    @Override
+    public Document toDocument(Manufacturer entity, NitriteMapper nitriteMapper) {
+        return Document.createDocument("name", entity.getName())
+            .put("address", entity.getAddress())
+            .put("uniqueId", entity.getUniqueId());
+    }
+
+    @Override
+    public Manufacturer fromDocument(Document document, NitriteMapper nitriteMapper) {
+        Manufacturer manufacturer = new Manufacturer();
+        manufacturer.setName(document.get("name", String.class));
+        manufacturer.setAddress(document.get("address", String.class));
+        manufacturer.setUniqueId(document.get("uniqueId", Integer.class));
+        return manufacturer;
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java
new file mode 100644
index 000000000..8d066491a
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ManufacturerDecorator.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import org.dizitart.no2.repository.EntityIndex;
+import org.dizitart.no2.repository.EntityDecorator;
+import org.dizitart.no2.repository.EntityId;
+
+import java.util.List;
+
+public class ManufacturerDecorator implements EntityDecorator {
+    @Override
+    public Class getEntityType() {
+        return Manufacturer.class;
+    }
+
+    @Override
+    public EntityId getIdField() {
+        return null;
+    }
+
+    @Override
+    public List getIndexFields() {
+        return null;
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java
new file mode 100644
index 000000000..89b332c8a
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/MiniProduct.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import lombok.Data;
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+
+@Data
+public class MiniProduct {
+    private String uniqueId;
+    private String manufacturerName;
+    private Double price;
+
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return MiniProduct.class;
+        }
+
+        @Override
+        public Document toDocument(MiniProduct entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument()
+                .put("productId.uniqueId", entity.getUniqueId())
+                .put("manufacturer.name", entity.getManufacturerName())
+                .put("price", entity.getPrice());
+        }
+
+        @Override
+        public MiniProduct fromDocument(Document document, NitriteMapper nitriteMapper) {
+            MiniProduct entity = new MiniProduct();
+            entity.setUniqueId(document.get("productId.uniqueId", String.class));
+            entity.setManufacturerName(document.get("manufacturer.name", String.class));
+            entity.setPrice(document.get("price", Double.class));
+            return entity;
+        }
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java
new file mode 100644
index 000000000..ff59219f9
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/Product.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import lombok.Data;
+
+@Data
+public class Product {
+    private ProductId productId;
+    private Manufacturer manufacturer;
+    private String productName;
+    private Double price;
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java
new file mode 100644
index 000000000..0421c0cac
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductConverter.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+
+public class ProductConverter implements EntityConverter {
+    @Override
+    public Class getEntityType() {
+        return Product.class;
+    }
+
+    @Override
+    public Document toDocument(Product entity, NitriteMapper nitriteMapper) {
+        Document productId = nitriteMapper.convert(entity.getProductId(), Document.class);
+        Document manufacturer = nitriteMapper.convert(entity.getManufacturer(), Document.class);
+
+        return Document.createDocument()
+            .put("productId", productId)
+            .put("manufacturer", manufacturer)
+            .put("productName", entity.getProductName())
+            .put("price", entity.getPrice());
+    }
+
+    @Override
+    public Product fromDocument(Document document, NitriteMapper nitriteMapper) {
+        Product entity = new Product();
+        ProductId productId = nitriteMapper.convert(document.get("productId", Document.class), ProductId.class);
+        Manufacturer manufacturer = nitriteMapper.convert(document.get("manufacturer", Document.class),
+            Manufacturer.class);
+        entity.setProductId(productId);
+        entity.setManufacturer(manufacturer);
+        entity.setProductName(document.get("productName", String.class));
+        entity.setPrice(document.get("price", Double.class));
+        return entity;
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java
new file mode 100644
index 000000000..180296343
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductDecorator.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import org.dizitart.no2.repository.EntityIndex;
+import org.dizitart.no2.index.IndexType;
+import org.dizitart.no2.repository.EntityDecorator;
+import org.dizitart.no2.repository.EntityId;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class ProductDecorator implements EntityDecorator {
+    @Override
+    public Class getEntityType() {
+        return Product.class;
+    }
+
+    @Override
+    public EntityId getIdField() {
+        return new EntityId("productId", "uniqueId", "productCode");
+    }
+
+    @Override
+    public List getIndexFields() {
+        return Arrays.asList(
+            new EntityIndex(IndexType.NON_UNIQUE, "manufacturer.name"),
+            new EntityIndex(IndexType.UNIQUE, "productName", "manufacturer.uniqueId")
+        );
+    }
+
+    @Override
+    public String getEntityName() {
+        return "product";
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java
new file mode 100644
index 000000000..dd2f9f04e
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductId.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import lombok.Data;
+
+@Data
+public class ProductId {
+    private String uniqueId;
+    private String productCode;
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java
new file mode 100644
index 000000000..2c02fddc0
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/repository/decorator/ProductIdConverter.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.integration.repository.decorator;
+
+import org.dizitart.no2.collection.Document;
+import org.dizitart.no2.common.mapper.EntityConverter;
+import org.dizitart.no2.common.mapper.NitriteMapper;
+
+public class ProductIdConverter implements EntityConverter {
+    @Override
+    public Class getEntityType() {
+        return ProductId.class;
+    }
+
+    @Override
+    public Document toDocument(ProductId entity, NitriteMapper nitriteMapper) {
+        return Document.createDocument("uniqueId", entity.getUniqueId())
+            .put("productCode", entity.getProductCode());
+    }
+
+    @Override
+    public ProductId fromDocument(Document document, NitriteMapper nitriteMapper) {
+        ProductId entity = new ProductId();
+        entity.setUniqueId(document.get("uniqueId", String.class));
+        entity.setProductCode(document.get("productCode", String.class));
+        return entity;
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java
index 05a2b095b..931f70a14 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionCollectionTest.java
@@ -21,7 +21,7 @@
 import org.dizitart.no2.integration.collection.BaseCollectionTest;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteCollection;
-import org.dizitart.no2.collection.meta.Attributes;
+import org.dizitart.no2.common.meta.Attributes;
 import org.dizitart.no2.exceptions.NitriteIOException;
 import org.dizitart.no2.exceptions.TransactionException;
 import org.dizitart.no2.index.IndexType;
@@ -299,11 +299,11 @@ public void testRollbackClear() {
                 txCol.insert(document2);
                 collection.insert(document2);
 
-                throw new TransactionException("failed");
+                transaction.commit();
             } catch (TransactionException e) {
                 assert transaction != null;
                 transaction.rollback();
-                assertEquals(2, collection.size());
+                assertEquals(0, collection.size());
             }
         }
     }
@@ -431,7 +431,7 @@ public void testCommitDropCollection() {
                 boolean expectedException = false;
                 try {
                     assertEquals(0, txCol.size());
-                } catch (NitriteIOException e) {
+                } catch (TransactionException e) {
                     expectedException = true;
                 }
                 assertTrue(expectedException);
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java
index 4e506726d..12251a1b2 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TransactionRepositoryTest.java
@@ -20,7 +20,7 @@
 import com.github.javafaker.Faker;
 import org.dizitart.no2.collection.Document;
 import org.dizitart.no2.collection.NitriteCollection;
-import org.dizitart.no2.collection.meta.Attributes;
+import org.dizitart.no2.common.meta.Attributes;
 import org.dizitart.no2.exceptions.NitriteIOException;
 import org.dizitart.no2.exceptions.TransactionException;
 import org.dizitart.no2.index.IndexType;
@@ -324,11 +324,11 @@ public void testRollbackClear() {
                 repository.insert(txData2);
 
                 transaction.commit();
-                fail();
             } catch (TransactionException e) {
                 assert transaction != null;
                 transaction.rollback();
-                assertEquals(2, repository.size());
+                // clear/drop can't be rolled back
+                assertEquals(0, repository.size());
             }
         }
     }
@@ -454,7 +454,7 @@ public void testCommitDropRepository() {
                 boolean expectedException = false;
                 try {
                     assertEquals(0, txRepo.size());
-                } catch (NitriteIOException e) {
+                } catch (TransactionException e) {
                     expectedException = true;
                 }
                 assertTrue(expectedException);
diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TxData.java b/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TxData.java
index 1e5ed7ca6..d9dd2d334 100644
--- a/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TxData.java
+++ b/nitrite/src/test/java/org/dizitart/no2/integration/transaction/TxData.java
@@ -21,7 +21,7 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.common.mapper.Mappable;
+import org.dizitart.no2.common.mapper.EntityConverter;
 import org.dizitart.no2.common.mapper.NitriteMapper;
 import org.dizitart.no2.repository.annotations.Id;
 
@@ -31,20 +31,31 @@
 @Data
 @NoArgsConstructor
 @AllArgsConstructor
-class TxData implements Mappable {
+public class TxData {
     @Id
     private Long id;
     private String name;
 
-    @Override
-    public Document write(NitriteMapper mapper) {
-        return Document.createDocument("id", id)
-            .put("name", name);
-    }
 
-    @Override
-    public void read(NitriteMapper mapper, Document document) {
-        id = document.get("id", Long.class);
-        name = document.get("name", String.class);
+    public static class Converter implements EntityConverter {
+
+        @Override
+        public Class getEntityType() {
+            return TxData.class;
+        }
+
+        @Override
+        public Document toDocument(TxData entity, NitriteMapper nitriteMapper) {
+            return Document.createDocument("id", entity.id)
+                .put("name", entity.name);
+        }
+
+        @Override
+        public TxData fromDocument(Document document, NitriteMapper nitriteMapper) {
+            TxData entity = new TxData();
+            entity.id = document.get("id", Long.class);
+            entity.name = document.get("name", String.class);
+            return entity;
+        }
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/migration/NitriteInstructionsTest.java b/nitrite/src/test/java/org/dizitart/no2/migration/NitriteInstructionSetTest.java
similarity index 85%
rename from nitrite/src/test/java/org/dizitart/no2/migration/NitriteInstructionsTest.java
rename to nitrite/src/test/java/org/dizitart/no2/migration/NitriteInstructionSetTest.java
index 0b1e7c621..32858c257 100644
--- a/nitrite/src/test/java/org/dizitart/no2/migration/NitriteInstructionsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/migration/NitriteInstructionSetTest.java
@@ -23,11 +23,11 @@
 
 import static org.junit.Assert.assertSame;
 
-public class NitriteInstructionsTest {
+public class NitriteInstructionSetTest {
     @Test
     public void testConstructor() {
         LinkedList migrationStepList = new LinkedList<>();
-        assertSame(migrationStepList, (new NitriteInstructions(migrationStepList)).getMigrationSteps());
+        assertSame(migrationStepList, (new NitriteInstructionSet(migrationStepList)).getMigrationSteps());
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/repository/AnnotationScannerTest.java b/nitrite/src/test/java/org/dizitart/no2/repository/AnnotationScannerTest.java
index 615c43162..1f3ca80a0 100644
--- a/nitrite/src/test/java/org/dizitart/no2/repository/AnnotationScannerTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/repository/AnnotationScannerTest.java
@@ -56,7 +56,7 @@ public void testScanIndices() {
         Class type = Object.class;
         AnnotationScanner annotationScanner = new AnnotationScanner(type, null,
                 new NitriteBuilderTest.CustomNitriteMapper());
-        annotationScanner.scanIndices();
+        annotationScanner.performScan();
         assertNull(annotationScanner.getObjectIdField());
     }
 
@@ -65,7 +65,7 @@ public void testScanIndices2() {
         Class type = Field.class;
         AnnotationScanner annotationScanner = new AnnotationScanner(type, null,
                 new NitriteBuilderTest.CustomNitriteMapper());
-        annotationScanner.scanIndices();
+        annotationScanner.performScan();
         assertNull(annotationScanner.getObjectIdField());
     }
 
@@ -75,7 +75,7 @@ public void testScanIndices3() {
         NitriteCollection collection = mock(NitriteCollection.class);
         AnnotationScanner annotationScanner = new AnnotationScanner(type, collection,
                 new NitriteBuilderTest.CustomNitriteMapper());
-        annotationScanner.scanIndices();
+        annotationScanner.performScan();
         assertNull(annotationScanner.getObjectIdField());
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/repository/DefaultObjectRepositoryTest.java b/nitrite/src/test/java/org/dizitart/no2/repository/DefaultObjectRepositoryTest.java
index 726cce323..071f4a747 100644
--- a/nitrite/src/test/java/org/dizitart/no2/repository/DefaultObjectRepositoryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/repository/DefaultObjectRepositoryTest.java
@@ -18,12 +18,9 @@
 package org.dizitart.no2.repository;
 
 import org.dizitart.no2.NitriteConfig;
-import org.dizitart.no2.collection.Document;
-import org.dizitart.no2.collection.FindOptions;
-import org.dizitart.no2.collection.NitriteCollection;
-import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.collection.*;
 import org.dizitart.no2.collection.events.CollectionEventListener;
-import org.dizitart.no2.collection.meta.Attributes;
+import org.dizitart.no2.common.meta.Attributes;
 import org.dizitart.no2.common.RecordStream;
 import org.dizitart.no2.common.WriteResult;
 import org.dizitart.no2.common.processors.ProcessorChain;
@@ -55,18 +52,6 @@ public void testAddProcessor() {
         assertNull(defaultObjectRepository.getAttributes());
     }
 
-    @Test
-    public void testRemoveProcessor() {
-        NitriteCollection nitriteCollection = mock(NitriteCollection.class);
-        doNothing().when(nitriteCollection).removeProcessor(any());
-        Class type = Object.class;
-        DefaultObjectRepository defaultObjectRepository = new DefaultObjectRepository<>(type,
-            nitriteCollection, new NitriteConfig());
-        defaultObjectRepository.removeProcessor(new ProcessorChain());
-        verify(nitriteCollection).removeProcessor(any());
-        assertNull(defaultObjectRepository.getAttributes());
-    }
-
     @Test
     public void testCreateIndex() {
         NitriteCollection nitriteCollection = mock(NitriteCollection.class);
@@ -170,9 +155,9 @@ public void testInsert() {
     public void testUpdate2() {
         DefaultObjectRepository defaultObjectRepository = (DefaultObjectRepository) mock(
             DefaultObjectRepository.class);
-        when(defaultObjectRepository.update(any(), (Object) any(), anyBoolean())).thenReturn(null);
-        defaultObjectRepository.update(mock(Filter.class), "Update", true);
-        verify(defaultObjectRepository).update(any(), (Object) any(), anyBoolean());
+        when(defaultObjectRepository.update(any(), (Object) any(), any())).thenReturn(null);
+        defaultObjectRepository.update(mock(Filter.class), "Update", UpdateOptions.updateOptions(true));
+        verify(defaultObjectRepository).update(any(), (Object) any(), any());
     }
 
     @Test
diff --git a/nitrite/src/test/java/org/dizitart/no2/repository/EntityDecoratorScannerTest.java b/nitrite/src/test/java/org/dizitart/no2/repository/EntityDecoratorScannerTest.java
new file mode 100644
index 000000000..5d6b6ccc0
--- /dev/null
+++ b/nitrite/src/test/java/org/dizitart/no2/repository/EntityDecoratorScannerTest.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2017-2022 Nitrite author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.dizitart.no2.repository;
+
+import lombok.Data;
+import org.dizitart.no2.Nitrite;
+import org.dizitart.no2.collection.NitriteCollection;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
+import org.dizitart.no2.index.IndexType;
+import org.dizitart.no2.integration.repository.data.ClassA;
+import org.dizitart.no2.integration.repository.data.ClassBConverter;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.*;
+
+public class EntityDecoratorScannerTest {
+
+    private EntityDecoratorScanner reader;
+    private NitriteCollection collection;
+
+    @Before
+    public void setUp() {
+        SimpleDocumentMapper nitriteMapper = new SimpleDocumentMapper();
+        nitriteMapper.registerEntityConverter(new ClassA.ClassAConverter());
+        nitriteMapper.registerEntityConverter(new ClassBConverter());
+        collection = Nitrite.builder().fieldSeparator(".").openOrCreate().getCollection("test");
+        reader = new EntityDecoratorScanner(new EntityADecorator(), collection, nitriteMapper);
+    }
+
+    @Test
+    public void testReadEntity() {
+        assertNull(reader.getObjectIdField());
+        assertTrue(reader.getIndices().isEmpty());
+
+        reader.readEntity();
+        assertNotNull(reader.getObjectIdField());
+        assertFalse(reader.getIndices().isEmpty());
+
+        ObjectIdField idField = reader.getObjectIdField();
+        assertEquals(idField.getIdFieldName(), "id");
+        assertArrayEquals(idField.getFieldNames(), new String[] {"uid", "string"});
+        assertArrayEquals(idField.getEmbeddedFieldNames(), new String[] {"id.uid", "id.string"});
+
+        Set entityIndexFields = reader.getIndices();
+        assertEquals(entityIndexFields.size(), 2);
+    }
+
+    @Test
+    public void testCreateIndices() {
+        assertFalse(collection.hasIndex("name", "age"));
+        assertFalse(collection.hasIndex("address"));
+        assertFalse(collection.hasIndex("id.uid", "id.string"));
+
+        reader.readEntity();
+        reader.createIndices();
+
+        assertTrue(collection.hasIndex("name", "age"));
+        assertTrue(collection.hasIndex("address"));
+        assertFalse(collection.hasIndex("id.uid", "id.string"));
+    }
+
+    @Test
+    public void testCreateIdIndex() {
+        assertNull(reader.getObjectIdField());
+        assertFalse(collection.hasIndex("name", "age"));
+        assertFalse(collection.hasIndex("address"));
+        assertFalse(collection.hasIndex("id.uid", "id.string"));
+
+        reader.readEntity();
+        reader.createIdIndex();
+
+        assertNotNull(reader.getObjectIdField());
+        assertFalse(collection.hasIndex("name", "age"));
+        assertFalse(collection.hasIndex("address"));
+        assertTrue(collection.hasIndex("id.uid", "id.string"));
+    }
+
+    @Data
+    public static class EntityA {
+        private ClassA id;
+        private String name;
+        private String age;
+        private String address;
+    }
+
+    public static class EntityADecorator implements EntityDecorator {
+
+        @Override
+        public Class getEntityType() {
+            return EntityA.class;
+        }
+
+        @Override
+        public EntityId getIdField() {
+            return new EntityId("id", "uid", "string");
+        }
+
+        @Override
+        public List getIndexFields() {
+            List list = new ArrayList<>();
+            EntityIndex fieldsA = new EntityIndex(IndexType.UNIQUE, "name", "age");
+            EntityIndex fieldsB = new EntityIndex(IndexType.NON_UNIQUE, "address");
+            list.add(fieldsA);
+            list.add(fieldsB);
+            return list;
+        }
+
+        @Override
+        public String getEntityName() {
+            return "a";
+        }
+    }
+}
diff --git a/nitrite/src/test/java/org/dizitart/no2/repository/IndexValidatorTest.java b/nitrite/src/test/java/org/dizitart/no2/repository/IndexValidatorTest.java
index 0c812ed53..7fd054b15 100644
--- a/nitrite/src/test/java/org/dizitart/no2/repository/IndexValidatorTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/repository/IndexValidatorTest.java
@@ -29,7 +29,7 @@ public class IndexValidatorTest {
 
     @Test
     public void testValidate() {
-        IndexValidator indexValidator = new IndexValidator(new Reflector());
+        IndexValidator indexValidator = new IndexValidator();
         Class fieldType = Object.class;
         assertThrows(IndexingException.class,
             () -> indexValidator.validate(fieldType, "Field", new NitriteBuilderTest.CustomNitriteMapper()));
@@ -37,7 +37,7 @@ public void testValidate() {
 
     @Test
     public void testValidate3() {
-        IndexValidator indexValidator = new IndexValidator(new Reflector());
+        IndexValidator indexValidator = new IndexValidator();
         Class fieldType = Field.class;
         assertThrows(IndexingException.class,
             () -> indexValidator.validate(fieldType, "Field", new NitriteBuilderTest.CustomNitriteMapper()));
@@ -45,7 +45,7 @@ public void testValidate3() {
 
     @Test
     public void testValidate4() {
-        IndexValidator indexValidator = new IndexValidator(new Reflector());
+        IndexValidator indexValidator = new IndexValidator();
         Class fieldType = Object.class;
         assertThrows(IndexingException.class, () -> indexValidator.validate(fieldType, "invalid type specified ",
             new NitriteBuilderTest.CustomNitriteMapper()));
diff --git a/nitrite/src/test/java/org/dizitart/no2/repository/ObjectCursorTest.java b/nitrite/src/test/java/org/dizitart/no2/repository/ObjectCursorTest.java
index b2edd5e3e..4ec228f17 100644
--- a/nitrite/src/test/java/org/dizitart/no2/repository/ObjectCursorTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/repository/ObjectCursorTest.java
@@ -23,9 +23,10 @@
 import org.dizitart.no2.collection.NitriteId;
 import org.dizitart.no2.common.Lookup;
 import org.dizitart.no2.common.RecordStream;
-import org.dizitart.no2.common.mapper.MappableMapper;
+import org.dizitart.no2.common.mapper.SimpleDocumentMapper;
 import org.dizitart.no2.common.processors.ProcessorChain;
 import org.dizitart.no2.common.streams.DocumentStream;
+import org.dizitart.no2.common.streams.MutatedObjectStream;
 import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.junit.Test;
@@ -80,9 +81,7 @@ public void testProject() {
 
     @Test
     public void testProject3() {
-        Class forNameResult = Object.class;
-        Class forNameResult1 = Object.class;
-        MappableMapper nitriteMapper = new MappableMapper(forNameResult, forNameResult1, Object.class);
+        SimpleDocumentMapper nitriteMapper = new SimpleDocumentMapper();
         RecordStream> recordStream = (RecordStream>) mock(
             RecordStream.class);
         DocumentStream cursor = new DocumentStream(recordStream, new ProcessorChain());
diff --git a/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryFactoryTest.java b/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryFactoryTest.java
index 27eeed486..56e799c12 100644
--- a/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryFactoryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryFactoryTest.java
@@ -23,6 +23,9 @@
 import org.dizitart.no2.exceptions.ValidationException;
 import org.junit.Test;
 
+import java.util.Date;
+import java.util.List;
+
 import static org.junit.Assert.assertThrows;
 
 public class RepositoryFactoryTest {
@@ -36,7 +39,7 @@ public void testGetRepository() {
     @Test
     public void testGetRepository2() {
         RepositoryFactory repositoryFactory = new RepositoryFactory(new CollectionFactory(new LockService()));
-        assertThrows(ValidationException.class, () -> repositoryFactory.getRepository(new NitriteConfig(), null));
+        assertThrows(ValidationException.class, () -> repositoryFactory.getRepository(new NitriteConfig(), (Class) null));
     }
 
     @Test
@@ -49,7 +52,39 @@ public void testGetRepository3() {
     public void testGetRepository4() {
         RepositoryFactory repositoryFactory = new RepositoryFactory(new CollectionFactory(new LockService()));
         assertThrows(ValidationException.class,
-            () -> repositoryFactory.getRepository(new NitriteConfig(), null, "Key"));
+            () -> repositoryFactory.getRepository(new NitriteConfig(), (Class) null, "Key"));
+    }
+
+    @Test
+    public void testGetRepository5() {
+        RepositoryFactory repositoryFactory = new RepositoryFactory(new CollectionFactory(new LockService()));
+        assertThrows(ValidationException.class,
+            () -> repositoryFactory.getRepository(new NitriteConfig(), (EntityDecorator) null, "Key"));
+    }
+
+    @Test
+    public void testGetRepository6() {
+        RepositoryFactory repositoryFactory = new RepositoryFactory(new CollectionFactory(new LockService()));
+        assertThrows(ValidationException.class,
+            () -> repositoryFactory.getRepository(null, new DemoDecorator(), "Key"));
+    }
+
+    public static class DemoDecorator implements EntityDecorator {
+
+        @Override
+        public Class getEntityType() {
+            return Date.class;
+        }
+
+        @Override
+        public EntityId getIdField() {
+            return null;
+        }
+
+        @Override
+        public List getIndexFields() {
+            return null;
+        }
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryOperationsTest.java b/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryOperationsTest.java
index df3580ea8..2aff316cd 100644
--- a/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryOperationsTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/repository/RepositoryOperationsTest.java
@@ -27,6 +27,7 @@
 import org.dizitart.no2.common.streams.DocumentStream;
 import org.dizitart.no2.common.tuples.Pair;
 import org.dizitart.no2.exceptions.NotIdentifiableException;
+import org.dizitart.no2.exceptions.ObjectMappingException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.dizitart.no2.filters.Filter;
 import org.junit.Test;
@@ -42,7 +43,7 @@ public void testConstructor() {
             () -> new RepositoryOperations(type, new NitriteBuilderTest.CustomNitriteMapper(), null));
     }
 
-    @Test
+    @Test(expected = ObjectMappingException.class)
     public void testToDocuments() {
         Class type = Object.class;
         assertEquals(3,
@@ -61,7 +62,7 @@ public void testToDocuments3() {
             .toDocuments(new Object[]{}));
     }
 
-    @Test
+    @Test(expected = ObjectMappingException.class)
     public void testToDocument() {
         Class type = Object.class;
         assertNull(
diff --git a/nitrite/src/test/java/org/dizitart/no2/store/StoreCatalogTest.java b/nitrite/src/test/java/org/dizitart/no2/store/StoreCatalogTest.java
index ded81e974..c16a665e4 100644
--- a/nitrite/src/test/java/org/dizitart/no2/store/StoreCatalogTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/store/StoreCatalogTest.java
@@ -17,12 +17,12 @@
 
 package org.dizitart.no2.store;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import org.dizitart.no2.store.memory.InMemoryStore;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 public class StoreCatalogTest {
     @Test
     public void testConstructor() {
@@ -48,7 +48,7 @@ public void testWriteRepositoryEntry() {
     @Test
     public void testWriteKeyedRepositoryEntries() {
         StoreCatalog storeCatalog = new StoreCatalog(new InMemoryStore());
-        storeCatalog.writeKeyedRepositoryEntries("Name");
+        storeCatalog.writeKeyedRepositoryEntry("Name");
         assertTrue(storeCatalog.getCollectionNames().isEmpty());
     }
 
@@ -69,12 +69,6 @@ public void testGetKeyedRepositoryNames() {
 
     @Test
     public void testRemove() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
         (new StoreCatalog(new InMemoryStore())).remove("Name");
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/store/UserAuthenticationServiceTest.java b/nitrite/src/test/java/org/dizitart/no2/store/UserAuthenticationServiceTest.java
index aa32c4958..e746d73ca 100644
--- a/nitrite/src/test/java/org/dizitart/no2/store/UserAuthenticationServiceTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/store/UserAuthenticationServiceTest.java
@@ -17,71 +17,33 @@
 
 package org.dizitart.no2.store;
 
-import org.dizitart.no2.exceptions.NitriteSecurityException;
 import org.dizitart.no2.store.memory.InMemoryStore;
 import org.junit.Test;
 
-import static org.junit.Assert.assertThrows;
-
 public class UserAuthenticationServiceTest {
     @Test
     public void testConstructor() {
-        // TODO: This test is incomplete.
-        //   Reason: Nothing to assert: the constructed class does not have observers (e.g. getters or public fields).
-        //   Add observers (e.g. getters or public fields) to the class.
-        //   See https://diff.blue/R002
-
         new UserAuthenticationService(null);
     }
 
-    @Test
-    public void testAuthenticate() {
-        assertThrows(NitriteSecurityException.class,
-            () -> (new UserAuthenticationService(new InMemoryStore())).authenticate("janedoe", "iloveyou", true));
-    }
-
     @Test
     public void testAuthenticate2() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
-        (new UserAuthenticationService(new InMemoryStore())).authenticate("", "iloveyou", true);
+        (new UserAuthenticationService(new InMemoryStore())).authenticate("", "iloveyou");
     }
 
     @Test
     public void testAuthenticate3() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
-        (new UserAuthenticationService(new InMemoryStore())).authenticate("janedoe", "", true);
+        (new UserAuthenticationService(new InMemoryStore())).authenticate("janedoe", "");
     }
 
     @Test
     public void testAuthenticate4() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
-        (new UserAuthenticationService(new InMemoryStore())).authenticate("janedoe", "iloveyou", false);
+        (new UserAuthenticationService(new InMemoryStore())).authenticate("janedoe", "iloveyou");
     }
 
     @Test
     public void testAuthenticate5() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
-        (new UserAuthenticationService(new InMemoryStore())).authenticate("", "iloveyou", false);
+        (new UserAuthenticationService(new InMemoryStore())).authenticate("", "iloveyou");
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryMapTest.java b/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryMapTest.java
index 4bf390ec8..1cf618589 100644
--- a/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryMapTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryMapTest.java
@@ -1,5 +1,6 @@
 package org.dizitart.no2.store.memory;
 
+import org.dizitart.no2.exceptions.InvalidOperationException;
 import org.dizitart.no2.exceptions.ValidationException;
 import org.junit.Test;
 
@@ -333,7 +334,7 @@ public void testIsEmpty2() {
         assertFalse(inMemoryMap.isEmpty());
     }
 
-    @Test
+    @Test(expected = InvalidOperationException.class)
     public void testDrop() {
         InMemoryMap inMemoryMap = new InMemoryMap<>("Map Name", new InMemoryStore());
         inMemoryMap.put("Key", "Value");
@@ -342,7 +343,7 @@ public void testDrop() {
         assertEquals(4, inMemoryMap.getAttributes().getAttributes().size());
     }
 
-    @Test
+    @Test(expected = InvalidOperationException.class)
     public void testDrop2() {
         InMemoryMap inMemoryMap = new InMemoryMap<>("42", new InMemoryStore());
         inMemoryMap.put("Key", "Value");
@@ -351,7 +352,7 @@ public void testDrop2() {
         assertEquals(4, inMemoryMap.getAttributes().getAttributes().size());
     }
 
-    @Test
+    @Test(expected = InvalidOperationException.class)
     public void testDrop3() {
         InMemoryMap inMemoryMap = new InMemoryMap<>("Map Name", new InMemoryStore());
         inMemoryMap.drop();
@@ -361,12 +362,6 @@ public void testDrop3() {
 
     @Test
     public void testClose() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
         (new InMemoryMap<>("Map Name", null)).close();
     }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryRTreeTest.java b/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryRTreeTest.java
index 445657d80..5dd54ecab 100644
--- a/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryRTreeTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/store/memory/InMemoryRTreeTest.java
@@ -1,5 +1,6 @@
 package org.dizitart.no2.store.memory;
 
+import org.dizitart.no2.common.util.SpatialKey;
 import org.junit.Test;
 
 import static org.junit.Assert.assertArrayEquals;
@@ -19,7 +20,7 @@ public void testSize() {
     @Test
     public void testSpatialKeyConstructor() {
         float[] floatArray = new float[]{10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f};
-        new InMemoryRTree.SpatialKey(123L, floatArray);
+        new SpatialKey(123L, floatArray);
         assertEquals(8, floatArray.length);
         assertArrayEquals(new float[]{10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f}, floatArray, 0.0f);
     }
diff --git a/nitrite/src/test/java/org/dizitart/no2/transaction/ChangeTypeTest.java b/nitrite/src/test/java/org/dizitart/no2/transaction/ChangeTypeTest.java
index 599c37904..a5ba03e6f 100644
--- a/nitrite/src/test/java/org/dizitart/no2/transaction/ChangeTypeTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/transaction/ChangeTypeTest.java
@@ -23,14 +23,9 @@
 
 public class ChangeTypeTest {
 
-    @Test
-    public void testValueOf3() {
-        assertEquals(ChangeType.AddProcessor, ChangeType.valueOf("AddProcessor"));
-    }
-
     @Test
     public void testValues() {
-        assertEquals(12, ChangeType.values().length);
+        assertEquals(10, ChangeType.values().length);
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalCollectionTest.java b/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalCollectionTest.java
index c09fcc21d..332f03377 100644
--- a/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalCollectionTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalCollectionTest.java
@@ -33,7 +33,7 @@ public void testConstructor() {
         TransactionContext transactionContext = new TransactionContext();
         transactionContext.setConfig(transactionConfig);
         assertThrows(NotIdentifiableException.class,
-            () -> new DefaultTransactionalCollection(null, transactionContext, null));
+            () -> new DefaultTransactionalCollection(null, transactionContext));
         verify(transactionConfig).getNitriteStore();
     }
 
@@ -47,14 +47,13 @@ public void testConstructor2() {
         TransactionContext transactionContext = new TransactionContext();
         transactionContext.setConfig(transactionConfig);
         DefaultTransactionalCollection actualDefaultTransactionalCollection = new DefaultTransactionalCollection(null,
-            transactionContext, null);
+            transactionContext);
         assertNull(actualDefaultTransactionalCollection.getCollectionName());
         assertFalse(actualDefaultTransactionalCollection.isDropped());
         TransactionContext transactionContext1 = actualDefaultTransactionalCollection.getTransactionContext();
         assertSame(transactionContext, transactionContext1);
         assertSame(transactionStore, actualDefaultTransactionalCollection.getStore());
         assertNull(actualDefaultTransactionalCollection.getPrimary());
-        assertNull(actualDefaultTransactionalCollection.getNitrite());
         assertNull(actualDefaultTransactionalCollection.getNitriteMap());
         assertNull(actualDefaultTransactionalCollection.getCollectionOperations().getAttributes());
         verify(transactionConfig, times(2)).getNitriteStore();
diff --git a/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalRepositoryTest.java b/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalRepositoryTest.java
index efa732946..993407d1f 100644
--- a/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalRepositoryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/transaction/DefaultTransactionalRepositoryTest.java
@@ -19,7 +19,7 @@
 
 import org.dizitart.no2.collection.FindOptions;
 import org.dizitart.no2.collection.events.CollectionEventListener;
-import org.dizitart.no2.collection.meta.Attributes;
+import org.dizitart.no2.common.meta.Attributes;
 import org.dizitart.no2.common.processors.ProcessorChain;
 import org.dizitart.no2.filters.Filter;
 import org.dizitart.no2.index.IndexOptions;
@@ -27,6 +27,7 @@
 
 import java.util.ArrayList;
 
+import static org.dizitart.no2.collection.UpdateOptions.updateOptions;
 import static org.mockito.Mockito.*;
 
 public class DefaultTransactionalRepositoryTest {
@@ -39,16 +40,6 @@ public void testAddProcessor() {
         verify(defaultTransactionalRepository).addProcessor(any());
     }
 
-    @Test
-    public void testRemoveProcessor() {
-        DefaultTransactionalRepository defaultTransactionalRepository = (DefaultTransactionalRepository) mock(
-            DefaultTransactionalRepository.class);
-        doNothing().when(defaultTransactionalRepository)
-            .removeProcessor(any());
-        defaultTransactionalRepository.removeProcessor(new ProcessorChain());
-        verify(defaultTransactionalRepository).removeProcessor(any());
-    }
-
     @Test
     public void testCreateIndex() {
         DefaultTransactionalRepository defaultTransactionalRepository = (DefaultTransactionalRepository) mock(
@@ -134,9 +125,9 @@ public void testUpdate() {
     public void testUpdate2() {
         DefaultTransactionalRepository defaultTransactionalRepository = (DefaultTransactionalRepository) mock(
             DefaultTransactionalRepository.class);
-        when(defaultTransactionalRepository.update(any(), (Object) any(), anyBoolean())).thenReturn(null);
-        defaultTransactionalRepository.update(mock(Filter.class), "Update", true);
-        verify(defaultTransactionalRepository).update(any(), (Object) any(), anyBoolean());
+        when(defaultTransactionalRepository.update(any(), (Object) any(), any())).thenReturn(null);
+        defaultTransactionalRepository.update(mock(Filter.class), "Update", updateOptions(true));
+        verify(defaultTransactionalRepository).update(any(), (Object) any(), any());
     }
 
     @Test
diff --git a/nitrite/src/test/java/org/dizitart/no2/transaction/JournalEntryTest.java b/nitrite/src/test/java/org/dizitart/no2/transaction/JournalEntryTest.java
index 831a9e6dc..8aa076efc 100644
--- a/nitrite/src/test/java/org/dizitart/no2/transaction/JournalEntryTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/transaction/JournalEntryTest.java
@@ -110,12 +110,6 @@ public void testHashCode() {
 
     @Test
     public void testHashCode2() {
-        // TODO: This test is incomplete.
-        //   Reason: No meaningful assertions found.
-        //   To help Diffblue Cover to find assertions, please add getters to the
-        //   class under test that return fields written by the method under test.
-        //   See https://diff.blue/R004
-
         (new JournalEntry(ChangeType.Insert, mock(Command.class), mock(Command.class))).hashCode();
     }
 }
diff --git a/nitrite/src/test/java/org/dizitart/no2/transaction/NitriteTransactionTest.java b/nitrite/src/test/java/org/dizitart/no2/transaction/NitriteTransactionTest.java
index 5e4c4c4be..6f5c48d0a 100644
--- a/nitrite/src/test/java/org/dizitart/no2/transaction/NitriteTransactionTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/transaction/NitriteTransactionTest.java
@@ -32,7 +32,7 @@ public void testConstructor() {
         Nitrite nitrite = mock(Nitrite.class);
         when(nitrite.getConfig()).thenReturn(new NitriteConfig());
         doReturn(new TransactionStore<>(new InMemoryStore())).when(nitrite).getStore();
-        assertEquals(State.Active, (new NitriteTransaction(nitrite, new LockService())).getState());
+        assertEquals(TransactionState.Active, (new NitriteTransaction(nitrite, new LockService())).getState());
         verify(nitrite).getConfig();
         verify(nitrite).getStore();
     }
diff --git a/nitrite/src/test/java/org/dizitart/no2/transaction/StateTest.java b/nitrite/src/test/java/org/dizitart/no2/transaction/TransactionStateTest.java
similarity index 58%
rename from nitrite/src/test/java/org/dizitart/no2/transaction/StateTest.java
rename to nitrite/src/test/java/org/dizitart/no2/transaction/TransactionStateTest.java
index ba93373cc..a5940c749 100644
--- a/nitrite/src/test/java/org/dizitart/no2/transaction/StateTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/transaction/TransactionStateTest.java
@@ -21,22 +21,22 @@
 
 import static org.junit.Assert.assertEquals;
 
-public class StateTest {
+public class TransactionStateTest {
     @Test
     public void testValueOf2() {
-        assertEquals(State.Aborted, State.valueOf("Aborted"));
+        assertEquals(TransactionState.Aborted, TransactionState.valueOf("Aborted"));
     }
 
     @Test
     public void testValues() {
-        State[] actualValuesResult = State.values();
+        TransactionState[] actualValuesResult = TransactionState.values();
         assertEquals(6, actualValuesResult.length);
-        assertEquals(State.Active, actualValuesResult[0]);
-        assertEquals(State.PartiallyCommitted, actualValuesResult[1]);
-        assertEquals(State.Committed, actualValuesResult[2]);
-        assertEquals(State.Closed, actualValuesResult[3]);
-        assertEquals(State.Failed, actualValuesResult[4]);
-        assertEquals(State.Aborted, actualValuesResult[5]);
+        assertEquals(TransactionState.Active, actualValuesResult[0]);
+        assertEquals(TransactionState.PartiallyCommitted, actualValuesResult[1]);
+        assertEquals(TransactionState.Committed, actualValuesResult[2]);
+        assertEquals(TransactionState.Closed, actualValuesResult[3]);
+        assertEquals(TransactionState.Failed, actualValuesResult[4]);
+        assertEquals(TransactionState.Aborted, actualValuesResult[5]);
     }
 }
 
diff --git a/nitrite/src/test/java/org/dizitart/no2/transaction/TransactionalMapTest.java b/nitrite/src/test/java/org/dizitart/no2/transaction/TransactionalMapTest.java
index 4687b5e6a..30ca37e64 100644
--- a/nitrite/src/test/java/org/dizitart/no2/transaction/TransactionalMapTest.java
+++ b/nitrite/src/test/java/org/dizitart/no2/transaction/TransactionalMapTest.java
@@ -435,7 +435,7 @@ public void testDrop() {
         transactionalMap.drop();
         assertTrue(transactionalMap.entries().toList().isEmpty());
         assertEquals(0L, transactionalMap.size());
-        assertEquals(4, transactionalMap.getAttributes().getAttributes().size());
+        assertNull(transactionalMap.getAttributes());
     }
 
     @Test
diff --git a/potassium-nitrite/build.gradle b/potassium-nitrite/build.gradle
index beadd0ae9..7fa336727 100644
--- a/potassium-nitrite/build.gradle
+++ b/potassium-nitrite/build.gradle
@@ -16,7 +16,6 @@
 
 buildscript {
     repositories {
-        jcenter()
         mavenCentral()
     }
 
@@ -37,7 +36,6 @@ plugins {
 }
 
 repositories {
-    jcenter()
     mavenCentral()
 }
 
@@ -59,11 +57,14 @@ dependencies {
 
     testApi project(':nitrite-mvstore-adapter')
     testImplementation "junit:junit:4.13.2"
-    testImplementation "joda-time:joda-time:2.10.13"
-    testImplementation "org.threeten:threetenbp:1.5.2"
-    testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.14.1"
-    testImplementation "org.apache.logging.log4j:log4j-core:2.15.0"
-    testImplementation "com.github.javafaker:javafaker:1.0.2"
+    testImplementation "joda-time:joda-time:2.10.14"
+    testImplementation 'org.threeten:threetenbp:1.6.0'
+    testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2"
+    testImplementation "org.apache.logging.log4j:log4j-core:2.17.2"
+    testImplementation ("com.github.javafaker:javafaker:1.0.2") {
+        exclude module: 'snakeyaml'
+    }
+    testImplementation 'org.yaml:snakeyaml:1.30'
 }
 
 compileKotlin {
diff --git a/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/Builder.kt b/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/Builder.kt
index b8ccac4cd..e0461e8e5 100644
--- a/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/Builder.kt
+++ b/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/Builder.kt
@@ -73,7 +73,7 @@ class Builder internal constructor() {
 /**
  * Opens or creates a new database. If it is an in-memory store, then it
  * will create a new one. If it is a file based store, and if the file does not
- * exists, then it will create a new file store and open; otherwise it will
+ * exist, then it will create a new file store and open; otherwise it will
  * open the existing file store.
  *
  * @param [userId] the user id
diff --git a/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2JacksonMapper.kt b/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2JacksonMapper.kt
index 308ac61d1..cf823a79b 100644
--- a/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2JacksonMapper.kt
+++ b/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2JacksonMapper.kt
@@ -16,20 +16,14 @@
 
 package org.dizitart.kno2
 
+import com.fasterxml.jackson.databind.Module
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.SerializationFeature
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
 import com.fasterxml.jackson.module.kotlin.KotlinModule
 import org.dizitart.no2.common.mapper.JacksonMapper
-import org.dizitart.no2.common.mapper.JacksonExtension
-import org.dizitart.no2.spatial.mapper.GeometryExtension
-import java.time.ZoneId
-import java.time.chrono.ChronoPeriod
-import java.time.temporal.Temporal
-import java.time.temporal.TemporalAdjuster
-import java.time.temporal.TemporalAmount
-import java.util.*
+import org.dizitart.no2.spatial.mapper.GeometryModule
 
 /**
  * Default [JacksonMapper] for potassium nitrite.
@@ -38,25 +32,18 @@ import java.util.*
  * @author Stefan Mandel
  * @since 2.1.0
  */
-open class KNO2JacksonMapper(vararg extensions: JacksonExtension) : JacksonMapper(GeometryExtension(), *extensions) {
+open class KNO2JacksonMapper(private vararg val modules: Module) : JacksonMapper() {
 
-    override fun createObjectMapper(): ObjectMapper {
-        // add value types from JavaTimeModule
-        addValueType(Temporal::class.java)
-        addValueType(TemporalAdjuster::class.java)
-        addValueType(TemporalAmount::class.java)
-        addValueType(ChronoPeriod::class.java)
-        addValueType(ZoneId::class.java)
-
-        // add value types from Jdk8Module
-        addValueType(OptionalInt::class.java)
-        addValueType(OptionalLong::class.java)
-        addValueType(OptionalDouble::class.java)
-
-        val objectMapper = super.createObjectMapper()
+    override fun getObjectMapper(): ObjectMapper {
+        val objectMapper = super.getObjectMapper()
         objectMapper.registerModule(KotlinModule.Builder().build())
         objectMapper.registerModule(Jdk8Module())
         objectMapper.registerModule(JavaTimeModule())
+        objectMapper.registerModule(GeometryModule())
+        for (module in modules) {
+            objectMapper.registerModule(module)
+        }
+
         objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
 
         return objectMapper
diff --git a/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2Module.kt b/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2Module.kt
index c7cce31f1..247a53775 100644
--- a/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2Module.kt
+++ b/potassium-nitrite/src/main/kotlin/org/dizitart/kno2/KNO2Module.kt
@@ -16,8 +16,8 @@
 
 package org.dizitart.kno2
 
+import com.fasterxml.jackson.databind.Module
 import org.dizitart.no2.common.util.Iterables.setOf
-import org.dizitart.no2.common.mapper.JacksonExtension
 import org.dizitart.no2.common.module.NitriteModule
 import org.dizitart.no2.common.module.NitritePlugin
 import org.dizitart.no2.spatial.SpatialIndexer
@@ -26,7 +26,7 @@ import org.dizitart.no2.spatial.SpatialIndexer
  *
  * @author Anindya Chatterjee
  */
-open class KNO2Module(private vararg val extensions: JacksonExtension) : NitriteModule {
+open class KNO2Module(private vararg val extensions: Module) : NitriteModule {
 
     override fun plugins(): MutableSet {
         return setOf(KNO2JacksonMapper(*extensions), SpatialIndexer())
diff --git a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/BackportJavaTimeTest.kt b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/BackportJavaTimeTest.kt
index ef8aef206..58c8888dd 100644
--- a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/BackportJavaTimeTest.kt
+++ b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/BackportJavaTimeTest.kt
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.module.SimpleModule
 import org.dizitart.no2.index.IndexType
 import org.dizitart.no2.repository.annotations.Id
 import org.dizitart.no2.repository.annotations.Index
-import org.dizitart.no2.common.mapper.JacksonExtension
 import org.dizitart.no2.mvstore.MVStoreModule
 import org.junit.Test
 import org.threeten.bp.LocalDateTime
@@ -39,42 +38,34 @@ import java.util.*
 class BackportJavaTimeTest {
     private val dbPath = getRandomTempDbFile()
 
-    @Index(value = ["time"], type = IndexType.NON_UNIQUE)
+    @Index(fields = ["time"], type = IndexType.NON_UNIQUE)
     data class TestData(
             @Id val id: String = UUID.randomUUID().toString(),
-            val time: LocalDateTime
+            val time: LocalDateTime = LocalDateTime.now()
     )
 
-    class ThreeTenAbpExtension : JacksonExtension {
-        override fun getModule(): Module {
-            return object : SimpleModule() {
-                override fun setupModule(context: SetupContext?) {
-                    addDeserializer(LocalDateTime::class.java, object : JsonDeserializer() {
-                        override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): LocalDateTime? {
-                            val timeStamp = p?.longValue
-                            return if (timeStamp == -1L || timeStamp == null) null else {
-                                LocalDateTime.ofEpochSecond(timeStamp, 0, ZoneOffset.UTC)
-                            }
-                        }
-                    })
-
-                    addSerializer(LocalDateTime::class.java, object : JsonSerializer() {
-                        override fun serialize(value: LocalDateTime?, gen: JsonGenerator?, serializers: SerializerProvider?) {
-                            if (value == null) {
-                                gen?.writeNull()
-                            } else {
-                                val timeStamp = value.toEpochSecond(ZoneOffset.UTC)
-                                gen?.writeNumber(timeStamp)
-                            }
-                        }
-                    })
-                    super.setupModule(context)
+    class ThreeTenAbpModule : SimpleModule() {
+        override fun setupModule(context: SetupContext?) {
+            addDeserializer(LocalDateTime::class.java, object : JsonDeserializer() {
+                override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): LocalDateTime? {
+                    val timeStamp = p?.longValue
+                    return if (timeStamp == -1L || timeStamp == null) null else {
+                        LocalDateTime.ofEpochSecond(timeStamp, 0, ZoneOffset.UTC)
+                    }
                 }
-            }
-        }
+            })
 
-        override fun getSupportedTypes(): List> {
-            return listOf(LocalDateTime::class.java)
+            addSerializer(LocalDateTime::class.java, object : JsonSerializer() {
+                override fun serialize(value: LocalDateTime?, gen: JsonGenerator?, serializers: SerializerProvider?) {
+                    if (value == null) {
+                        gen?.writeNull()
+                    } else {
+                        val timeStamp = value.toEpochSecond(ZoneOffset.UTC)
+                        gen?.writeNumber(timeStamp)
+                    }
+                }
+            })
+            super.setupModule(context)
         }
     }
 
@@ -82,7 +73,7 @@ class BackportJavaTimeTest {
     fun testIssue59() {
         val db = nitrite {
             loadModule(MVStoreModule(dbPath))
-            loadModule(KNO2Module(ThreeTenAbpExtension()))
+            loadModule(KNO2Module(ThreeTenAbpModule()))
         }
 
         val repo = db.getRepository()
diff --git a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/FilterTest.kt b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/FilterTest.kt
index 3286e8f67..48fcf26ef 100644
--- a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/FilterTest.kt
+++ b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/FilterTest.kt
@@ -322,7 +322,7 @@ class FilterTest : BaseTest() {
     @Test
     fun testBetweenFilter() {
         @Getter
-        class TestData(private val age: Date)
+        class TestData(private val age: Date = Date())
 
         val data1 = TestData(GregorianCalendar(2020, Calendar.JANUARY, 11).time)
         val data2 = TestData(GregorianCalendar(2021, Calendar.FEBRUARY, 12).time)
diff --git a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/NitriteTest.kt b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/NitriteTest.kt
index de873a1b9..fc56686f5 100644
--- a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/NitriteTest.kt
+++ b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/NitriteTest.kt
@@ -17,12 +17,10 @@
 package org.dizitart.kno2
 
 import org.dizitart.kno2.filters.text
-import org.dizitart.no2.collection.FindOptions
 import org.dizitart.no2.collection.FindOptions.orderBy
 import org.dizitart.no2.collection.FindOptions.skipBy
 import org.dizitart.no2.common.SortOrder
 import org.dizitart.no2.exceptions.UniqueConstraintException
-import org.dizitart.no2.index.IndexOptions
 import org.dizitart.no2.index.IndexOptions.indexOptions
 import org.dizitart.no2.index.IndexType
 import org.dizitart.no2.mvstore.MVStoreModule
@@ -191,7 +189,7 @@ interface MyInterface {
     val id: UUID
 }
 
-@Indices(value = [(Index(value = ["name"], type = IndexType.NON_UNIQUE))])
+@Indices(value = [(Index(fields = ["name"], type = IndexType.NON_UNIQUE))])
 abstract class SomeAbsClass(
         @Id override val id: UUID = UUID.randomUUID(),
         open val name: String = "abcd"
@@ -201,29 +199,29 @@ abstract class SomeAbsClass(
 
 @InheritIndices
 class MyClass(
-    override val id: UUID,
-    override val name: String,
-    override val checked: Boolean) : SomeAbsClass(id, name)
+    override val id: UUID = UUID.randomUUID(),
+    override val name: String = "",
+    override val checked: Boolean = false) : SomeAbsClass(id, name)
 
 @InheritIndices
 class MyClass2(
-    override val id: UUID,
-    override val name: String,
-    override val checked: Boolean,
-    val importance: Int
+    override val id: UUID = UUID.randomUUID(),
+    override val name: String = "",
+    override val checked: Boolean = false,
+    val importance: Int = 0
 ) : SomeAbsClass(id, name)
 
 data class CaObject(
-        @Id val localId: UUID,
-        val name: String
+        @Id val localId: UUID = UUID.randomUUID(),
+        val name: String = ""
 )
 
-@Indices(value = [(Index(value = ["time"], type = IndexType.UNIQUE))])
+@Indices(value = [(Index(fields = ["time"], type = IndexType.UNIQUE))])
 data class ClassWithLocalDateTime(
-    val name: String,
-    val time: LocalDateTime
+    val name: String = "",
+    val time: LocalDateTime = LocalDateTime.now()
 )
 
-data class NestedObjects(var ob1: String, @Id val id: String, val list: List)
-data class TempObject(val name: String, val aga: Int, val add: LevelUnder)
-data class LevelUnder(val street: String, val number: Int)
\ No newline at end of file
+data class NestedObjects(var ob1: String = "", @Id val id: String = "", val list: List = listOf())
+data class TempObject(val name: String = "", val aga: Int = 0, val add: LevelUnder = LevelUnder())
+data class LevelUnder(val street: String = "", val number: Int = 0)
\ No newline at end of file
diff --git a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/ObjectFilterTest.kt b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/ObjectFilterTest.kt
index 0aa0b6990..fabb10706 100644
--- a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/ObjectFilterTest.kt
+++ b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/ObjectFilterTest.kt
@@ -321,19 +321,19 @@ class ObjectFilterTest : BaseTest() {
     }
 }
 
-@Indices(Index(value = ["text"], type = IndexType.FULL_TEXT))
-data class TestData(@Id val id: Int, val text: String, val list: List = listOf())
+@Indices(Index(fields = ["text"], type = IndexType.FULL_TEXT))
+data class TestData(@Id val id: Int = 0, val text: String = "", val list: List = listOf())
 
-class ListData(val name: String, val score: Int)
+class ListData(val name: String = "", val score: Int = 0)
 
 data class SimpleObject(
-        @Id val id: UUID,
-        val value: Boolean
+    @Id val id: UUID = UUID.randomUUID(),
+    val value: Boolean = false
 )
 
-@Index(value = ["geometry"], type = SpatialIndexer.SPATIAL_INDEX)
+@Index(fields = ["geometry"], type = SpatialIndexer.SPATIAL_INDEX)
 data class SpatialData(
-        @Id val id: Long,
-        val geometry: Geometry
+        @Id val id: Long = 0,
+        val geometry: Geometry? = null
 )
 
diff --git a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/TransactionTest.kt b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/TransactionTest.kt
index 8ec173dc8..82b8f3966 100644
--- a/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/TransactionTest.kt
+++ b/potassium-nitrite/src/test/kotlin/org/dizitart/kno2/TransactionTest.kt
@@ -20,7 +20,7 @@ package org.dizitart.kno2
 import com.github.javafaker.Faker
 import org.dizitart.no2.collection.Document
 import org.dizitart.no2.collection.UpdateOptions
-import org.dizitart.no2.collection.meta.Attributes
+import org.dizitart.no2.common.meta.Attributes
 import org.dizitart.no2.exceptions.NitriteIOException
 import org.dizitart.no2.exceptions.TransactionException
 import org.dizitart.no2.filters.FluentFilter
@@ -485,7 +485,7 @@ class TransactionTest: BaseTest() {
                 var expectedException = false
                 try {
                     Assert.assertEquals(0, txCol.size())
-                } catch (e: NitriteIOException) {
+                } catch (e: TransactionException) {
                     expectedException = true
                 }
                 Assert.assertTrue(expectedException)