From ad5873cf0b8935f1d7756c5019ae2a0dc111677f Mon Sep 17 00:00:00 2001 From: Joshua Lockerman <> Date: Wed, 8 Jul 2020 20:09:36 -0400 Subject: [PATCH 1/2] Genericize the cache by using interfaces --- cache.go | 81 ++++++++++++++++++++++++------- cache_test.go | 36 +++++++------- list.go | 130 ++++++++++++++------------------------------------ 3 files changed, 118 insertions(+), 129 deletions(-) diff --git a/cache.go b/cache.go index 45fd473..0dddea4 100644 --- a/cache.go +++ b/cache.go @@ -1,6 +1,7 @@ package smolcache import ( + "encoding/binary" "sync" "sync/atomic" @@ -36,13 +37,13 @@ type Interner struct { type block struct { lock sync.RWMutex // guarded by lock - elements map[string]*Element + elements map[interface{}]*Element // only safe to not use a pointer since blocks never move sweep List // CLOCK sweep state, guarded by clockLock - next *Element + prev *Element // pad blocks out to be cache aligned _padding [16]byte } @@ -54,7 +55,19 @@ func WithMax(max uint64) *Interner { } } -func (i *Interner) Insert(key string, value int64) { +func WithMaxAndShards(max uint64, shards int) *Interner { + //TODO variable number of shards + return &Interner{ + max: max, + seed: maphash.MakeSeed(), + } +} + +func (i *Interner) GetSeed() maphash.Seed { + return i.seed +} + +func (i *Interner) InsertString(key string, value interface{}) { newSize := atomic.AddUint64(&i.count, 1) needsEvict := newSize > i.max if needsEvict { @@ -64,16 +77,34 @@ func (i *Interner) Insert(key string, value int64) { h := maphash.Hash{} h.SetSeed(i.seed) h.WriteString(key) - blockNum := h.Sum64() % 127 + i.InsertWithHash(key, value, h.Sum64()) +} + +func (i *Interner) InsertInt(key uint64, value interface{}) { + newSize := atomic.AddUint64(&i.count, 1) + needsEvict := newSize > i.max + if needsEvict { + i.evict() + } + + h := maphash.Hash{} + h.SetSeed(i.seed) + b := [8]byte{} + binary.LittleEndian.PutUint64(b[:], key) + i.InsertWithHash(key, value, h.Sum64()) +} + +func (i *Interner) InsertWithHash(key interface{}, value interface{}, hash uint64) { + blockNum := hash % 127 block := &i.maps[blockNum] block.insert(key, value) } -func (b *block) insert(key string, value int64) bool { +func (b *block) insert(key interface{}, value interface{}) bool { b.lock.Lock() defer b.lock.Unlock() if b.elements == nil { - b.elements = make(map[string]*Element) + b.elements = make(map[interface{}]*Element) } _, present := b.elements[key] if present { @@ -106,23 +137,25 @@ func (i *Interner) evict() { func (b *block) tryEvict() (evicted bool, reachedEnd bool) { b.lock.Lock() defer b.lock.Unlock() - if b.next == nil { - b.next = b.sweep.Front() - if b.next == nil { - return false, true - } + if b.prev == nil { + b.prev = b.sweep.Root() + } + + elem := b.prev.Next() + if elem == nil { + return false, true } evicted = false reachedEnd = false for !evicted && !reachedEnd { - elem := b.next - b.next = elem.Next() - reachedEnd = b.next == nil if elem.used != 0 { elem.used = 0 + b.prev = elem + elem = b.prev.Next() + reachedEnd = elem == nil } else { - key, _ := b.sweep.Remove(elem) + key, _ := b.sweep.RemoveNext(b.prev) delete(b.elements, key) evicted = true } @@ -131,16 +164,28 @@ func (b *block) tryEvict() (evicted bool, reachedEnd bool) { return evicted, reachedEnd } -func (i *Interner) Get(key string) (int64, bool) { +func (i *Interner) GetString(key string) (interface{}, bool) { h := maphash.Hash{} h.SetSeed(i.seed) h.WriteString(key) - blockNum := h.Sum64() % 127 + return i.GetWithHash(key, h.Sum64()) +} + +func (i *Interner) GetInt(key uint64) (interface{}, bool) { + h := maphash.Hash{} + h.SetSeed(i.seed) + b := [8]byte{} + binary.LittleEndian.PutUint64(b[:], key) + return i.GetWithHash(key, h.Sum64()) +} + +func (i *Interner) GetWithHash(key interface{}, hash uint64) (interface{}, bool) { + blockNum := hash % 127 block := &i.maps[blockNum] return block.get(key) } -func (b *block) get(key string) (int64, bool) { +func (b *block) get(key interface{}) (interface{}, bool) { b.lock.RLock() defer b.lock.RUnlock() if b.elements == nil { diff --git a/cache_test.go b/cache_test.go index 56fa0a8..42e579b 100644 --- a/cache_test.go +++ b/cache_test.go @@ -13,8 +13,8 @@ func TestWriteAndGetOnCache(t *testing.T) { cache := WithMax(100) - cache.Insert("1", 1) - val, found := cache.Get("1") + cache.InsertString("1", 1) + val, found := cache.GetString("1") // then if !found { @@ -30,14 +30,14 @@ func TestEntryNotFound(t *testing.T) { cache := WithMax(100) - val, found := cache.Get("nonExistingKey") + val, found := cache.GetString("nonExistingKey") if found { t.Errorf("found %d for noexistent key", val) } - cache.Insert("key", 1) + cache.InsertString("key", 1) - val, found = cache.Get("nonExistingKey") + val, found = cache.GetString("nonExistingKey") if found { t.Errorf("found %d for noexistent key", val) } @@ -49,18 +49,18 @@ func TestEviction(t *testing.T) { cache := WithMax(10) for i := 0; i < 10; i++ { key := fmt.Sprintf("%d", i) - cache.Insert(key, int64(i)) + cache.InsertString(key, int64(i)) if i != 5 { - cache.Get(key) + cache.GetString(key) } } - cache.Insert("100", 100) - cache.Get("100") + cache.InsertString("100", 100) + cache.GetString("100") for i := 0; i < 10; i++ { key := fmt.Sprintf("%d", i) - val, found := cache.Get(key) + val, found := cache.GetString(key) if i != 5 && (!found || val != int64(i)) { t.Errorf("missing value %d, got %d", i, val) } else if i == 5 && found { @@ -71,17 +71,17 @@ func TestEviction(t *testing.T) { } } - val, found := cache.Get("100") + val, found := cache.GetString("100") if !found || val != 100 { t.Errorf("missing value 100, got %d", val) } - cache.Insert("101", 101) - cache.Get("101") + cache.InsertString("101", 101) + cache.GetString("101") for i := 0; i < 10; i++ { key := fmt.Sprintf("%d", i) - val, found := cache.Get(key) + val, found := cache.GetString(key) if i != 5 && i != 2 && (!found || val != int64(i)) { t.Errorf("missing value %d, (found: %v) got %d", i, found, val) } else if (i == 5 || i == 2) && found { @@ -89,11 +89,11 @@ func TestEviction(t *testing.T) { } } - val, found = cache.Get("100") + val, found = cache.GetString("100") if !found || val != 100 { t.Errorf("missing value 100, got %d", val) } - val, found = cache.Get("101") + val, found = cache.GetString("101") if !found || val != 101 { t.Errorf("missing value 101, got %d", val) } @@ -110,7 +110,7 @@ func TestCacheGetRandomly(t *testing.T) { for i := 0; i < ntest; i++ { r := rand.Int63() % 20000 key := fmt.Sprintf("%d", r) - cache.Insert(key, r+1) + cache.InsertString(key, r+1) } wg.Done() }() @@ -118,7 +118,7 @@ func TestCacheGetRandomly(t *testing.T) { for i := 0; i < ntest; i++ { r := rand.Int63() key := fmt.Sprintf("%d", r) - if val, found := cache.Get(key); found && val != r+1 { + if val, found := cache.GetString(key); found && val != r+1 { t.Errorf("got %s ->\n %x\n expected:\n %x\n ", key, val, r+1) } } diff --git a/list.go b/list.go index e06c3a7..e133fb7 100644 --- a/list.go +++ b/list.go @@ -1,146 +1,90 @@ -// based on go std package "container/list", specialized to our -// usecase. original license: -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - package smolcache // Element is an element of a linked list. type Element struct { // The value stored with this element. - key string - Value int64 - - // pad Elements out to be cache aligned - _padding [8]byte + key interface{} + Value interface{} - // Next and previous pointers in the doubly-linked list of elements. + // Next and previous pointers in the singly-linked list of elements. // To simplify the implementation, internally a list l is implemented - // as a ring, such that &l.root is both the next element of the last - // list element (l.Back()) and the previous element of the first list - // element (l.Front()). - next, prev *Element - - // The list to which this element belongs. - list *List - + // as a ring + next *Element //CLOCK marker if this is recently used used uint32 + + // pad Elements out to be cache aligned + _padding [16]byte } // Next returns the next list element or nil. func (e *Element) Next() *Element { - if p := e.next; e.list != nil && p != &e.list.root { - return p - } - return nil -} - -// Prev returns the previous list element or nil. -func (e *Element) Prev() *Element { - if p := e.prev; e.list != nil && p != &e.list.root { - return p - } - return nil + return e.next } -// List represents a doubly linked list. -// The zero value for List is an empty list ready to use. +// List represents a singly linked list. type List struct { - root Element // sentinel list element, only &root, root.prev, and root.next are used - len int // current list length excluding (this) sentinel element + root Element // sentinel list element, only &root and root.next are used + last *Element } // Init initializes or clears list l. func (l *List) Init() *List { - l.root.next = &l.root - l.root.prev = &l.root - l.len = 0 + l.root.next = nil + l.last = &l.root return l } // New returns an initialized list. func New() *List { return new(List).Init() } -// Len returns the number of elements of list l. -// The complexity is O(1). -func (l *List) Len() int { return l.len } - -// Front returns the first element of list l or nil if the list is empty. -func (l *List) Front() *Element { - if l.len == 0 { - return nil - } - return l.root.next -} - -// lazyInit lazily initializes a zero List value. -func (l *List) lazyInit() { - if l.root.next == nil { +// Front returns the root element of list l. +func (l *List) Root() *Element { + if l.last == nil { l.Init() } + return &l.root } // insert inserts e after at, increments l.len, and returns e. func (l *List) insert(e, at *Element) *Element { n := at.next at.next = e - e.prev = at e.next = n - n.prev = e - e.list = l - l.len++ return e } // insertValue is a convenience wrapper for insert(&Element{Value: v}, at). -func (l *List) insertValue(k string, v int64, at *Element) *Element { +func (l *List) insertValue(k interface{}, v interface{}, at *Element) *Element { return l.insert(&Element{key: k, Value: v}, at) } // remove removes e from its list, decrements l.len, and returns e. -func (l *List) remove(e *Element) *Element { - e.prev.next = e.next - e.next.prev = e.prev - e.next = nil // avoid memory leaks - e.prev = nil // avoid memory leaks - e.list = nil - l.len-- - return e -} - -// move moves e to next to at and returns e. -func (l *List) move(e, at *Element) *Element { - if e == at { - return e - } - e.prev.next = e.next - e.next.prev = e.prev - - n := at.next - at.next = e - e.prev = at - e.next = n - n.prev = e - - return e +func (l *List) removeNext(e *Element) *Element { + next := e.next + e.next = e.next.next + next.next = nil // avoid memory leaks + return next } // Remove removes e from l if e is an element of list l. // It returns the element value e.Value. // The element must not be nil. -func (l *List) Remove(e *Element) (string, int64) { - if e.list == l { - // if e.list == l, l must have been initialized when e was inserted - // in l or l == nil (e is a zero Element) and l.remove will crash - l.remove(e) +func (l *List) RemoveNext(e *Element) (key interface{}, val interface{}) { + if e.next == nil { + return } - return e.key, e.Value + next := l.removeNext(e) + if next == l.last { + l.last = e + } + return next.key, next.Value } // PushBack inserts a new element e with value v at the back of list l and returns e. -func (l *List) PushBack(k string, v int64) *Element { - l.lazyInit() - return l.insertValue(k, v, l.root.prev) +func (l *List) PushBack(k interface{}, v interface{}) *Element { + if l.last == nil { + l.Init() + } + return l.insertValue(k, v, l.last) } From 9c71b4492be834d28bced2e24690db4972589fdc Mon Sep 17 00:00:00 2001 From: Joshua Lockerman <> Date: Thu, 9 Jul 2020 13:30:52 -0400 Subject: [PATCH 2/2] Use and array to store the elements instead of a list This seems to have slightly higher variance than the list version; it sometimes performs better on writes, sometimes a bit worse. --- cache.go | 64 +++++++++++++++++++++++++--------------- list.go | 90 -------------------------------------------------------- 2 files changed, 41 insertions(+), 113 deletions(-) delete mode 100644 list.go diff --git a/cache.go b/cache.go index 0dddea4..a955e69 100644 --- a/cache.go +++ b/cache.go @@ -36,16 +36,24 @@ type Interner struct { type block struct { lock sync.RWMutex - // guarded by lock - elements map[interface{}]*Element + // guarded by lock, stores indexes into storage + elements map[interface{}]int - // only safe to not use a pointer since blocks never move - sweep List + storage []Element + // CLOCK sweep state, index into storage guarded by clockLock + next int +} - // CLOCK sweep state, guarded by clockLock - prev *Element - // pad blocks out to be cache aligned - _padding [16]byte +type Element struct { + // The value stored with this element. + key interface{} + Value interface{} + + //CLOCK marker if this is recently used + used uint32 + + // pad Elements out to be cache aligned + _padding [24]byte } func WithMax(max uint64) *Interner { @@ -104,14 +112,15 @@ func (b *block) insert(key interface{}, value interface{}) bool { b.lock.Lock() defer b.lock.Unlock() if b.elements == nil { - b.elements = make(map[interface{}]*Element) + b.elements = make(map[interface{}]int) } _, present := b.elements[key] if present { return false } - elem := b.sweep.PushBack(key, value) - b.elements[key] = elem + index := len(b.storage) + b.storage = append(b.storage, Element{key: key, Value: value}) + b.elements[key] = index return true } @@ -137,28 +146,35 @@ func (i *Interner) evict() { func (b *block) tryEvict() (evicted bool, reachedEnd bool) { b.lock.Lock() defer b.lock.Unlock() - if b.prev == nil { - b.prev = b.sweep.Root() - } - - elem := b.prev.Next() - if elem == nil { + if len(b.storage) == 0 { return false, true } + if b.next >= len(b.storage) { + b.next = 0 + } evicted = false reachedEnd = false for !evicted && !reachedEnd { + elem := &b.storage[b.next] if elem.used != 0 { elem.used = 0 - b.prev = elem - elem = b.prev.Next() - reachedEnd = elem == nil } else { - key, _ := b.sweep.RemoveNext(b.prev) + key := elem.key delete(b.elements, key) + // if elem isn't the last element in storage, swap it + // with the last element + if b.next < len(b.storage)-1 { + *elem = b.storage[len(b.storage)-1] + b.elements[elem.key] = b.next + } + // pop off the last element + b.storage[len(b.storage)-1] = Element{} + b.storage = b.storage[:len(b.storage)-1] evicted = true } + b.next += 1 + reachedEnd = b.next >= len(b.storage) } return evicted, reachedEnd @@ -191,11 +207,12 @@ func (b *block) get(key interface{}) (interface{}, bool) { if b.elements == nil { return 0, false } - elem, present := b.elements[key] + idx, present := b.elements[key] if !present { return 0, false } + elem := &b.storage[idx] if atomic.LoadUint32(&elem.used) == 0 { atomic.StoreUint32(&elem.used, 1) } @@ -218,11 +235,12 @@ func (b *block) unmark(key string) bool { if b.elements == nil { return false } - elem, present := b.elements[key] + idx, present := b.elements[key] if !present { return false } + elem := &b.storage[idx] if atomic.LoadUint32(&elem.used) != 0 { atomic.StoreUint32(&elem.used, 0) } diff --git a/list.go b/list.go deleted file mode 100644 index e133fb7..0000000 --- a/list.go +++ /dev/null @@ -1,90 +0,0 @@ -package smolcache - -// Element is an element of a linked list. -type Element struct { - // The value stored with this element. - key interface{} - Value interface{} - - // Next and previous pointers in the singly-linked list of elements. - // To simplify the implementation, internally a list l is implemented - // as a ring - next *Element - //CLOCK marker if this is recently used - used uint32 - - // pad Elements out to be cache aligned - _padding [16]byte -} - -// Next returns the next list element or nil. -func (e *Element) Next() *Element { - return e.next -} - -// List represents a singly linked list. -type List struct { - root Element // sentinel list element, only &root and root.next are used - last *Element -} - -// Init initializes or clears list l. -func (l *List) Init() *List { - l.root.next = nil - l.last = &l.root - return l -} - -// New returns an initialized list. -func New() *List { return new(List).Init() } - -// Front returns the root element of list l. -func (l *List) Root() *Element { - if l.last == nil { - l.Init() - } - return &l.root -} - -// insert inserts e after at, increments l.len, and returns e. -func (l *List) insert(e, at *Element) *Element { - n := at.next - at.next = e - e.next = n - return e -} - -// insertValue is a convenience wrapper for insert(&Element{Value: v}, at). -func (l *List) insertValue(k interface{}, v interface{}, at *Element) *Element { - return l.insert(&Element{key: k, Value: v}, at) -} - -// remove removes e from its list, decrements l.len, and returns e. -func (l *List) removeNext(e *Element) *Element { - next := e.next - e.next = e.next.next - next.next = nil // avoid memory leaks - return next -} - -// Remove removes e from l if e is an element of list l. -// It returns the element value e.Value. -// The element must not be nil. -func (l *List) RemoveNext(e *Element) (key interface{}, val interface{}) { - if e.next == nil { - return - } - next := l.removeNext(e) - if next == l.last { - l.last = e - } - return next.key, next.Value -} - -// PushBack inserts a new element e with value v at the back of list l and returns e. -func (l *List) PushBack(k interface{}, v interface{}) *Element { - if l.last == nil { - l.Init() - } - return l.insertValue(k, v, l.last) -}