layout: pattern

title: Sharding

folder: sharding

permalink: /patterns/sharding/

categories: Other

tags:

- Java

- Difficulty-Beginner

Sharding pattern means divide the data store into horizontal partitions or shards. Each shard has the same schema, but holds its own distinct subset of the data.

A shard is a data store in its own right (it can contain the data for many entities of different types), running on a server acting as a storage node.

This pattern offers the following benefits:

- You can scale the system out by adding further shards running on additional storage nodes.

- A system can use off the shelf commodity hardware rather than specialized (and expensive) computers for each storage node.

- You can reduce contention and improved performance by balancing the workload across shards.

- In the cloud, shards can be located physically close to the users that will access the data.

* [Cloud Design Patterns: Prescriptive Architecture Guidance for Cloud Applications - Sharding Pattern](https://docs.microsoft.com/en-us/previous-versions/msp-n-p/dn589797(v=pandp.10)?redirectedfrom=MSDN)

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0"

xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

<parent>

<artifactId>java-design-patterns</artifactId>

<groupId>com.iluwatar</groupId>

<version>1.22.0-SNAPSHOT</version>

</parent>

<modelVersion>4.0.0</modelVersion>

<artifactId>sharding</artifactId>

<dependencies>

<dependency>

<groupId>junit</groupId>

<artifactId>junit</artifactId>

</dependency>

</dependencies>

</project>

package com.iluwatar.sharding;

public class App {

/**

public static void main(String[] args) {

var data1 = new Data(1, "data1", Data.DataType.type1);

var data2 = new Data(2, "data2", Data.DataType.type2);

var data3 = new Data(3, "data3", Data.DataType.type3);

var data4 = new Data(4, "data4", Data.DataType.type1);

var shard1 = new Shard(1);

var shard2 = new Shard(2);

var shard3 = new Shard(3);

ShardManager manager = new LookupShardManager();

manager.addNewShard(shard1);

manager.addNewShard(shard2);

manager.addNewShard(shard3);

manager.storeData(data1);

manager.storeData(data2);

manager.storeData(data3);

manager.storeData(data4);

shard1.clearData();

shard2.clearData();

shard3.clearData();

manager = new RangeShardManager();

manager.addNewShard(shard1);

manager.addNewShard(shard2);

manager.addNewShard(shard3);

manager.storeData(data1);

manager.storeData(data2);

manager.storeData(data3);

manager.storeData(data4);

shard1.clearData();

shard2.clearData();

shard3.clearData();

manager = new HashShardManager();

manager.addNewShard(shard1);

manager.addNewShard(shard2);

manager.addNewShard(shard3);

manager.storeData(data1);

manager.storeData(data2);

manager.storeData(data3);

manager.storeData(data4);

shard1.clearData();

shard2.clearData();

shard3.clearData();

}

}

package com.iluwatar.sharding;

public class Data {

private int key;

private String value;

private DataType type;

/**

public Data(final int key, final String value, final DataType type) {

this.key = key;

this.value = value;

this.type = type;

}

public int getKey() {

return key;

}

public void setKey(final int key) {

this.key = key;

}

public String getValue() {

return value;

}

public void setValue(final String value) {

this.value = value;

}

public DataType getType() {

return type;

}

public void setType(DataType type) {

this.type = type;

}

enum DataType {

type1, type2, type3

}

@Override

public String toString() {

return "Data {" + "key="

+ key + ", value='" + value

+ '\'' + ", type=" + type + '}';

}

}

package com.iluwatar.sharding;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

public class HashShardManager extends ShardManager {

private static final Logger LOGGER = LoggerFactory.getLogger(HashShardManager.class);

@Override

public int storeData(Data data) {

var shardId = allocateShard(data);

var shard = shardMap.get(shardId);

shard.storeData(data);

LOGGER.info(data.toString() + " is stored in Shard " + shardId);

return shardId;

}

@Override

protected int allocateShard(Data data) {

var shardCount = shardMap.size();

var hash = data.getKey() % shardCount;

return hash == 0 ? hash + shardCount : hash;

}

}

package com.iluwatar.sharding;

import java.util.HashMap;

import java.util.Map;

import java.util.Random;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

public class LookupShardManager extends ShardManager {

private static final Logger LOGGER = LoggerFactory.getLogger(LookupShardManager.class);

private Map<Integer, Integer> lookupMap = new HashMap<>();

@Override

public int storeData(Data data) {

var shardId = allocateShard(data);

lookupMap.put(data.getKey(), shardId);

var shard = shardMap.get(shardId);

shard.storeData(data);

LOGGER.info(data.toString() + " is stored in Shard " + shardId);

return shardId;

}

@Override

protected int allocateShard(Data data) {

var key = data.getKey();

if (lookupMap.containsKey(key)) {

return lookupMap.get(key);

} else {

var shardCount = shardMap.size();

var allocatedShardId = new Random().nextInt(shardCount - 1) + 1;

return allocatedShardId;

}

}

}