diff --git a/CHANGELOG.md b/CHANGELOG.md index 454f1709..c4c5b500 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # developing +- [Fix 5126: support bloom filter cache](https://github.com/beego/beego/pull/5126) - [Fix 5117: support write though cache](https://github.com/beego/beego/pull/5117) - [add read through for cache module](https://github.com/beego/beego/pull/5116) - [add singleflight cache for cache module](https://github.com/beego/beego/pull/5119) diff --git a/client/cache/bloom_filter_cache.go b/client/cache/bloom_filter_cache.go new file mode 100644 index 00000000..6dceb3af --- /dev/null +++ b/client/cache/bloom_filter_cache.go @@ -0,0 +1,71 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "context" + "errors" + "time" + + "github.com/beego/beego/v2/core/berror" +) + +type BloomFilterCache struct { + Cache + BloomFilter + loadFunc func(ctx context.Context, key string) (any, error) + expiration time.Duration // set cache expiration, default never expire +} + +type BloomFilter interface { + Test(data string) bool + Add(data string) +} + +func NewBloomFilterCache(cache Cache, ln func(context.Context, string) (any, error), blm BloomFilter, + expiration time.Duration, +) (*BloomFilterCache, error) { + if cache == nil || ln == nil || blm == nil { + return nil, berror.Error(InvalidInitParameters, "missing required parameters") + } + + return &BloomFilterCache{ + Cache: cache, + BloomFilter: blm, + loadFunc: ln, + expiration: expiration, + }, nil +} + +func (bfc *BloomFilterCache) Get(ctx context.Context, key string) (any, error) { + val, err := bfc.Cache.Get(ctx, key) + if err != nil && !errors.Is(err, ErrKeyNotExist) { + return nil, err + } + if errors.Is(err, ErrKeyNotExist) { + exist := bfc.BloomFilter.Test(key) + if exist { + val, err = bfc.loadFunc(ctx, key) + if err != nil { + return nil, berror.Wrap(err, LoadFuncFailed, "cache unable to load data") + } + err = bfc.Put(ctx, key, val, bfc.expiration) + if err != nil { + return val, err + } + } + } + return val, nil +} diff --git a/client/cache/bloom_filter_cache_test.go b/client/cache/bloom_filter_cache_test.go new file mode 100644 index 00000000..f160abbb --- /dev/null +++ b/client/cache/bloom_filter_cache_test.go @@ -0,0 +1,203 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint +package cache + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/beego/beego/v2/core/berror" + "github.com/bits-and-blooms/bloom/v3" + "github.com/stretchr/testify/assert" +) + +type MockDB struct { + Db Cache + loadCnt int64 +} + +type BloomFilterMock struct { + *bloom.BloomFilter + lock *sync.RWMutex + concurrent bool +} + +func (b *BloomFilterMock) Add(data string) { + if b.concurrent { + b.lock.Lock() + defer b.lock.Unlock() + } + b.BloomFilter.AddString(data) +} + +func (b *BloomFilterMock) Test(data string) bool { + if b.concurrent { + b.lock.Lock() + defer b.lock.Unlock() + } + return b.BloomFilter.TestString(data) +} + +var ( + mockDB = MockDB{Db: NewMemoryCache(), loadCnt: 0} + mockBloom = &BloomFilterMock{ + BloomFilter: bloom.NewWithEstimates(20000, 0.01), + lock: &sync.RWMutex{}, + concurrent: false, + } + loadFunc = func(ctx context.Context, key string) (any, error) { + mockDB.loadCnt += 1 // flag of number load data from db + v, err := mockDB.Db.Get(context.Background(), key) + if err != nil { + return nil, errors.New("fail") + } + return v, nil + } + cacheUnderlying = NewMemoryCache() +) + +func TestBloomFilterCache_Get(t *testing.T) { + testCases := []struct { + name string + key string + wantVal any + + before func() + after func() + + wantErrCode uint32 + }{ + // case: keys exist in cache + // want: not load data from db + { + name: "not_load_db", + before: func() { + _ = cacheUnderlying.Put(context.Background(), "exist_in_cache", "123", time.Minute) + }, + key: "exist_in_DB", + after: func() { + assert.Equal(t, mockDB.loadCnt, int64(0)) + _ = cacheUnderlying.Delete(context.Background(), "exist_in_cache") + mockDB.loadCnt = 0 + _ = mockDB.Db.ClearAll(context.Background()) + }, + }, + // case: keys not exist in cache, not exist in bloom + // want: not load data from db + { + name: "not_load_db", + before: func() { + _ = mockDB.Db.ClearAll(context.Background()) + _ = mockDB.Db.Put(context.Background(), "exist_in_DB", "exist_in_DB", 0) + mockBloom.AddString("other") + }, + key: "exist_in_DB", + after: func() { + assert.Equal(t, mockDB.loadCnt, int64(0)) + mockBloom.ClearAll() + mockDB.loadCnt = 0 + _ = mockDB.Db.ClearAll(context.Background()) + }, + }, + // case: keys not exist in cache, exist in bloom, exist in db, + // want: load data from db, and set cache + { + name: "load_db", + before: func() { + _ = mockDB.Db.ClearAll(context.Background()) + _ = mockDB.Db.Put(context.Background(), "exist_in_DB", "exist_in_DB", 0) + mockBloom.Add("exist_in_DB") + }, + key: "exist_in_DB", + wantVal: "exist_in_DB", + after: func() { + assert.Equal(t, mockDB.loadCnt, int64(1)) + _ = cacheUnderlying.Delete(context.Background(), "exist_in_DB") + mockBloom.ClearAll() + mockDB.loadCnt = 0 + _ = mockDB.Db.ClearAll(context.Background()) + }, + }, + // case: keys not exist in cache, exist in bloom, not exist in db, + // want: load func error + { + name: "load db fail", + before: func() { + mockBloom.Add("not_exist_in_DB") + }, + after: func() { + assert.Equal(t, mockDB.loadCnt, int64(1)) + mockBloom.ClearAll() + mockDB.loadCnt = 0 + _ = mockDB.Db.ClearAll(context.Background()) + }, + key: "not_exist_in_DB", + wantErrCode: LoadFuncFailed.Code(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.before() + bfc, err := NewBloomFilterCache(cacheUnderlying, loadFunc, mockBloom, time.Minute) + assert.Nil(t, err) + + got, err := bfc.Get(context.Background(), tc.key) + if tc.wantErrCode != 0 { + errCode, _ := berror.FromError(err) + assert.Equal(t, tc.wantErrCode, errCode.Code()) + return + } else { + assert.Nil(t, err) + } + assert.Equal(t, tc.wantVal, got) + + cacheVal, _ := bfc.Cache.Get(context.Background(), tc.key) + assert.Equal(t, tc.wantVal, cacheVal) + tc.after() + }) + } +} + +// This implementation of Bloom filters cache is NOT safe for concurrent use. +// Uncomment the following method. +// func TestBloomFilterCache_Get_Concurrency(t *testing.T) { +// bfc, err := NewBloomFilterCache(cacheUnderlying, loadFunc, mockBloom, time.Minute) +// assert.Nil(t, err) +// +// _ = mockDB.Db.ClearAll(context.Background()) +// _ = mockDB.Db.Put(context.Background(), "key_11", "value_11", 0) +// mockBloom.AddString("key_11") +// +// var wg sync.WaitGroup +// wg.Add(100000) +// for i := 0; i < 100000; i++ { +// key := fmt.Sprintf("key_%d", i) +// go func(key string) { +// defer wg.Done() +// val, _ := bfc.Get(context.Background(), key) +// +// if val != nil { +// assert.Equal(t, "value_11", val) +// } +// }(key) +// } +// wg.Wait() +// assert.Equal(t, int64(1), mockDB.loadCnt) +// } diff --git a/go.mod b/go.mod index ed16f7d9..5c8eb76e 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,8 @@ require ( require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.4.0 // indirect + github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect diff --git a/go.sum b/go.sum index 61a99479..a15fe700 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,11 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8= +github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= +github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/casbin/casbin v1.9.1 h1:ucjbS5zTrmSLtH4XogqOG920Poe6QatdXtz1FEbApeM= @@ -316,6 +321,7 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112 h1:NBrpnvz0pDPf3+HXZ1C9GcJd1DTpWDLcLWZhNq6uP7o= github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=