server/beego
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bloom filter cache (#5126)

Browse Source 
* feature: add bloom filter cache
This commit is contained in:
hookokoko 2022-12-26 23:53:27 +08:00 committed by Ming Deng
parent 0bee140abb
commit 72956feb26
5 changed files with 283 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -1,4 +1,5 @@
# developing
- [Fix 5126: support bloom filter cache](https://github.com/beego/beego/pull/5126)
- [Fix 5117: support write though cache](https://github.com/beego/beego/pull/5117)
- [add read through for cache module](https://github.com/beego/beego/pull/5116)
- [add singleflight cache for cache module](https://github.com/beego/beego/pull/5119)

71
client/cache/bloom_filter_cache.go vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2014 beego Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cache
import (
"context"
"errors"
"time"
"github.com/beego/beego/v2/core/berror"
)
type BloomFilterCache struct {
Cache
BloomFilter
loadFunc func(ctx context.Context, key string) (any, error)
expiration time.Duration // set cache expiration, default never expire
}
type BloomFilter interface {
Test(data string) bool
Add(data string)
}
func NewBloomFilterCache(cache Cache, ln func(context.Context, string) (any, error), blm BloomFilter,
expiration time.Duration,
) (*BloomFilterCache, error) {
if cache == nil || ln == nil || blm == nil {
return nil, berror.Error(InvalidInitParameters, "missing required parameters")
}
return &BloomFilterCache{
Cache: cache,
BloomFilter: blm,
loadFunc: ln,
expiration: expiration,
}, nil
}
func (bfc *BloomFilterCache) Get(ctx context.Context, key string) (any, error) {
val, err := bfc.Cache.Get(ctx, key)
if err != nil && !errors.Is(err, ErrKeyNotExist) {
return nil, err
}
if errors.Is(err, ErrKeyNotExist) {
exist := bfc.BloomFilter.Test(key)
if exist {
val, err = bfc.loadFunc(ctx, key)
if err != nil {
return nil, berror.Wrap(err, LoadFuncFailed, "cache unable to load data")
}
err = bfc.Put(ctx, key, val, bfc.expiration)
if err != nil {
return val, err
}
}
}
return val, nil
}

203
client/cache/bloom_filter_cache_test.go vendored Normal file
View File

@ -0,0 +1,203 @@
// Copyright 2014 beego Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// nolint
package cache
import (
"context"
"errors"
"sync"
"testing"
"time"
"github.com/beego/beego/v2/core/berror"
"github.com/bits-and-blooms/bloom/v3"
"github.com/stretchr/testify/assert"
)
type MockDB struct {
Db Cache
loadCnt int64
}
type BloomFilterMock struct {
*bloom.BloomFilter
lock *sync.RWMutex
concurrent bool
}
func (b *BloomFilterMock) Add(data string) {
if b.concurrent {
b.lock.Lock()
defer b.lock.Unlock()
}
b.BloomFilter.AddString(data)
}
func (b *BloomFilterMock) Test(data string) bool {
if b.concurrent {
b.lock.Lock()
defer b.lock.Unlock()
}
return b.BloomFilter.TestString(data)
}
var (
mockDB = MockDB{Db: NewMemoryCache(), loadCnt: 0}
mockBloom = &BloomFilterMock{
BloomFilter: bloom.NewWithEstimates(20000, 0.01),
lock: &sync.RWMutex{},
concurrent: false,
}
loadFunc = func(ctx context.Context, key string) (any, error) {
mockDB.loadCnt += 1 // flag of number load data from db
v, err := mockDB.Db.Get(context.Background(), key)
if err != nil {
return nil, errors.New("fail")
}
return v, nil
}
cacheUnderlying = NewMemoryCache()
)
func TestBloomFilterCache_Get(t *testing.T) {
testCases := []struct {
name string
key string
wantVal any
before func()
after func()
wantErrCode uint32
}{
// case: keys exist in cache
// want: not load data from db
{
name: "not_load_db",
before: func() {
_ = cacheUnderlying.Put(context.Background(), "exist_in_cache", "123", time.Minute)
},
key: "exist_in_DB",
after: func() {
assert.Equal(t, mockDB.loadCnt, int64(0))
_ = cacheUnderlying.Delete(context.Background(), "exist_in_cache")
mockDB.loadCnt = 0
_ = mockDB.Db.ClearAll(context.Background())
},
},
// case: keys not exist in cache, not exist in bloom
// want: not load data from db
{
name: "not_load_db",
before: func() {
_ = mockDB.Db.ClearAll(context.Background())
_ = mockDB.Db.Put(context.Background(), "exist_in_DB", "exist_in_DB", 0)
mockBloom.AddString("other")
},
key: "exist_in_DB",
after: func() {
assert.Equal(t, mockDB.loadCnt, int64(0))
mockBloom.ClearAll()
mockDB.loadCnt = 0
_ = mockDB.Db.ClearAll(context.Background())
},
},
// case: keys not exist in cache, exist in bloom, exist in db,
// want: load data from db, and set cache
{
name: "load_db",
before: func() {
_ = mockDB.Db.ClearAll(context.Background())
_ = mockDB.Db.Put(context.Background(), "exist_in_DB", "exist_in_DB", 0)
mockBloom.Add("exist_in_DB")
},
key: "exist_in_DB",
wantVal: "exist_in_DB",
after: func() {
assert.Equal(t, mockDB.loadCnt, int64(1))
_ = cacheUnderlying.Delete(context.Background(), "exist_in_DB")
mockBloom.ClearAll()
mockDB.loadCnt = 0
_ = mockDB.Db.ClearAll(context.Background())
},
},
// case: keys not exist in cache, exist in bloom, not exist in db,
// want: load func error
{
name: "load db fail",
before: func() {
mockBloom.Add("not_exist_in_DB")
},
after: func() {
assert.Equal(t, mockDB.loadCnt, int64(1))
mockBloom.ClearAll()
mockDB.loadCnt = 0
_ = mockDB.Db.ClearAll(context.Background())
},
key: "not_exist_in_DB",
wantErrCode: LoadFuncFailed.Code(),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
tc.before()
bfc, err := NewBloomFilterCache(cacheUnderlying, loadFunc, mockBloom, time.Minute)
assert.Nil(t, err)
got, err := bfc.Get(context.Background(), tc.key)
if tc.wantErrCode != 0 {
errCode, _ := berror.FromError(err)
assert.Equal(t, tc.wantErrCode, errCode.Code())
return
} else {
assert.Nil(t, err)
}
assert.Equal(t, tc.wantVal, got)
cacheVal, _ := bfc.Cache.Get(context.Background(), tc.key)
assert.Equal(t, tc.wantVal, cacheVal)
tc.after()
})
}
}
// This implementation of Bloom filters cache is NOT safe for concurrent use.
// Uncomment the following method.
// func TestBloomFilterCache_Get_Concurrency(t *testing.T) {
// bfc, err := NewBloomFilterCache(cacheUnderlying, loadFunc, mockBloom, time.Minute)
// assert.Nil(t, err)
//
// _ = mockDB.Db.ClearAll(context.Background())
// _ = mockDB.Db.Put(context.Background(), "key_11", "value_11", 0)
// mockBloom.AddString("key_11")
//
// var wg sync.WaitGroup
// wg.Add(100000)
// for i := 0; i < 100000; i++ {
// key := fmt.Sprintf("key_%d", i)
// go func(key string) {
// defer wg.Done()
// val, _ := bfc.Get(context.Background(), key)
//
// if val != nil {
// assert.Equal(t, "value_11", val)
// }
// }(key)
// }
// wg.Wait()
// assert.Equal(t, int64(1), mockDB.loadCnt)
// }

2
go.mod
View File

@ -45,6 +45,8 @@ require (
require (
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.4.0 // indirect
github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect

6
go.sum
View File

@ -52,6 +52,11 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8=
github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ=
github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/casbin/casbin v1.9.1 h1:ucjbS5zTrmSLtH4XogqOG920Poe6QatdXtz1FEbApeM=
@ -316,6 +321,7 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112 h1:NBrpnvz0pDPf3+HXZ1C9GcJd1DTpWDLcLWZhNq6uP7o=
github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=