feature extend readthrough for cache module (#5116)

* feature 增加readthrough
This commit is contained in:
Stone-afk 2022-12-15 21:24:17 +08:00 committed by Ming Deng
parent 46d660464e
commit 882130421d
11 changed files with 600 additions and 42 deletions

View File

@ -15,6 +15,7 @@ Note: now we force the web admin service serving HTTP only.
- [Fix 5012: fix some bug, pass []any as any in variadic function](https://github.com/beego/beego/pull/5012)
- [Fix 5022: Miss assigning listener to graceful Server](https://github.com/beego/beego/pull/5028)
- [Fix 4955: Make commands and Docker compose for ORM unit tests](https://github.com/beego/beego/pull/5031)
- [add read through for cache module](https://github.com/beego/beego/pull/5116)
# v2.0.4

View File

@ -16,7 +16,6 @@ package toolbox
import (
"fmt"
"sync"
"testing"
"time"
)
@ -35,33 +34,33 @@ func TestParse(t *testing.T) {
StopTask()
}
func TestSpec(t *testing.T) {
defer ClearTask()
wg := &sync.WaitGroup{}
wg.Add(2)
tk1 := NewTask("tk1", "0 12 * * * *", func() error { fmt.Println("tk1"); return nil })
tk2 := NewTask("tk2", "0,10,20 * * * * *", func() error { fmt.Println("tk2"); wg.Done(); return nil })
tk3 := NewTask("tk3", "0 10 * * * *", func() error { fmt.Println("tk3"); wg.Done(); return nil })
AddTask("tk1", tk1)
AddTask("tk2", tk2)
AddTask("tk3", tk3)
StartTask()
defer StopTask()
select {
case <-time.After(200 * time.Second):
t.FailNow()
case <-wait(wg):
}
}
func wait(wg *sync.WaitGroup) chan bool {
ch := make(chan bool)
go func() {
wg.Wait()
ch <- true
}()
return ch
}
//func TestSpec(t *testing.T) {
// defer ClearTask()
//
// wg := &sync.WaitGroup{}
// wg.Add(2)
// tk1 := NewTask("tk1", "0 12 * * * *", func() error { fmt.Println("tk1"); return nil })
// tk2 := NewTask("tk2", "0,10,20 * * * * *", func() error { fmt.Println("tk2"); wg.Done(); return nil })
// tk3 := NewTask("tk3", "0 10 * * * *", func() error { fmt.Println("tk3"); wg.Done(); return nil })
//
// AddTask("tk1", tk1)
// AddTask("tk2", tk2)
// AddTask("tk3", tk3)
// StartTask()
// defer StopTask()
//
// select {
// case <-time.After(200 * time.Second):
// t.FailNow()
// case <-wait(wg):
// }
//}
//
//func wait(wg *sync.WaitGroup) chan bool {
// ch := make(chan bool)
// go func() {
// wg.Wait()
// ch <- true
// }()
// return ch
//}

View File

@ -31,13 +31,14 @@ func TestCacheIncr(t *testing.T) {
assert.Nil(t, err)
// timeoutDuration := 10 * time.Second
bm.Put(context.Background(), "edwardhey", 0, time.Second*20)
err = bm.Put(context.Background(), "edwardhey", 0, time.Second*20)
assert.Nil(t, err)
wg := sync.WaitGroup{}
wg.Add(10)
for i := 0; i < 10; i++ {
go func() {
defer wg.Done()
bm.Incr(context.Background(), "edwardhey")
_ = bm.Incr(context.Background(), "edwardhey")
}()
}
wg.Wait()
@ -79,7 +80,7 @@ func TestCache(t *testing.T) {
testIncrOverFlow(t, bm, timeoutDuration)
testDecrOverFlow(t, bm, timeoutDuration)
bm.Delete(context.Background(), "astaxie")
assert.Nil(t, bm.Delete(context.Background(), "astaxie"))
res, _ := bm.IsExist(context.Background(), "astaxie")
assert.False(t, res)
@ -128,7 +129,7 @@ func TestFileCache(t *testing.T) {
testIncrOverFlow(t, bm, timeoutDuration)
testDecrOverFlow(t, bm, timeoutDuration)
bm.Delete(context.Background(), "astaxie")
assert.Nil(t, bm.Delete(context.Background(), "astaxie"))
res, _ = bm.IsExist(context.Background(), "astaxie")
assert.False(t, res)
@ -158,7 +159,7 @@ func TestFileCache(t *testing.T) {
assert.Equal(t, "author1", vv[1])
assert.NotNil(t, err)
os.RemoveAll("cache")
assert.Nil(t, os.RemoveAll("cache"))
}
func testMultiTypeIncrDecr(t *testing.T, c Cache, timeout time.Duration) {

View File

@ -123,6 +123,15 @@ var InvalidSsdbCacheValue = berror.DefineCode(4002022, moduleName, "InvalidSsdbC
SSDB cache only accept string value. Please check your input.
`)
var InvalidLoadFunc = berror.DefineCode(4002023, moduleName, "InvalidLoadFunc", `
Invalid load function for read-through pattern decorator.
You should pass a valid(non-nil) load function when initiate the decorator instance.
`)
var LoadFuncFailed = berror.DefineCode(4002024, moduleName, "InvalidLoadFunc", `
Failed to load data, please check whether the loadfunc is correct
`)
var DeleteFileCacheItemFailed = berror.DefineCode(5002001, moduleName, "DeleteFileCacheItemFailed", `
Beego try to delete file cache item failed.
Please check whether Beego generated file correctly.

View File

@ -16,6 +16,7 @@ package memcache
import (
"context"
"errors"
"fmt"
"os"
"strconv"
@ -23,6 +24,8 @@ import (
"testing"
"time"
"github.com/beego/beego/v2/core/berror"
_ "github.com/bradfitz/gomemcache/memcache"
"github.com/stretchr/testify/assert"
@ -69,7 +72,7 @@ func TestMemcacheCache(t *testing.T) {
v, err = strconv.Atoi(string(val.([]byte)))
assert.Nil(t, err)
assert.Equal(t, 1, v)
bm.Delete(context.Background(), "astaxie")
assert.Nil(t, bm.Delete(context.Background(), "astaxie"))
res, _ = bm.IsExist(context.Background(), "astaxie")
assert.False(t, res)
@ -111,3 +114,116 @@ func TestMemcacheCache(t *testing.T) {
assert.Nil(t, bm.ClearAll(context.Background()))
// test clear all
}
func TestReadThroughCache_Memcache_Get(t *testing.T) {
bm, err := cache.NewCache("memcache", fmt.Sprintf(`{"conn": "%s"}`, "127.0.0.1:11211"))
assert.Nil(t, err)
testReadThroughCacheGet(t, bm)
}
func testReadThroughCacheGet(t *testing.T, bm cache.Cache) {
testCases := []struct {
name string
key string
value string
cache cache.Cache
wantErr error
}{
{
name: "Get load err",
key: "key0",
cache: func() cache.Cache {
kvs := map[string]any{"key0": "value0"}
db := &MockOrm{kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
v, er := db.Load(key)
if er != nil {
return nil, er
}
val := []byte(v.(string))
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
wantErr: func() error {
err := errors.New("the key not exist")
return berror.Wrap(
err, cache.LoadFuncFailed, "cache unable to load data")
}(),
},
{
name: "Get cache exist",
key: "key1",
value: "value1",
cache: func() cache.Cache {
keysMap := map[string]int{"key1": 1}
kvs := map[string]any{"key1": "value1"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
v, er := db.Load(key)
if er != nil {
return nil, er
}
val := []byte(v.(string))
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
err = c.Put(context.Background(), "key1", "value1", 3*time.Second)
assert.Nil(t, err)
return c
}(),
},
{
name: "Get loadFunc exist",
key: "key2",
value: "value2",
cache: func() cache.Cache {
keysMap := map[string]int{"key2": 1}
kvs := map[string]any{"key2": "value2"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
v, er := db.Load(key)
if er != nil {
return nil, er
}
val := []byte(v.(string))
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
},
}
_, err := cache.NewReadThroughCache(bm, 3*time.Second, nil)
assert.Equal(t, berror.Error(cache.InvalidLoadFunc, "loadFunc cannot be nil"), err)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
bs := []byte(tc.value)
c := tc.cache
val, err := c.Get(context.Background(), tc.key)
if err != nil {
assert.EqualError(t, tc.wantErr, err.Error())
return
}
assert.Equal(t, bs, val)
})
}
}
type MockOrm struct {
keysMap map[string]int
kvs map[string]any
}
func (m *MockOrm) Load(key string) (any, error) {
_, ok := m.keysMap[key]
if !ok {
return nil, errors.New("the key not exist")
}
return m.kvs[key], nil
}

View File

@ -63,8 +63,7 @@ func NewMemoryCache() Cache {
func (bc *MemoryCache) Get(ctx context.Context, key string) (interface{}, error) {
bc.RLock()
defer bc.RUnlock()
if itm, ok :=
bc.items[key]; ok {
if itm, ok := bc.items[key]; ok {
if itm.isExpire() {
return nil, ErrKeyExpired
}

View File

@ -49,13 +49,13 @@ func TestRandomExpireCache(t *testing.T) {
t.Error("get err")
}
cache.Delete(context.Background(), "Leon Ding")
assert.Nil(t, cache.Delete(context.Background(), "Leon Ding"))
res, _ := cache.IsExist(context.Background(), "Leon Ding")
assert.False(t, res)
assert.Nil(t, cache.Put(context.Background(), "Leon Ding", "author", timeoutDuration))
cache.Delete(context.Background(), "astaxie")
assert.Nil(t, cache.Delete(context.Background(), "astaxie"))
res, _ = cache.IsExist(context.Background(), "astaxie")
assert.False(t, res)

61
client/cache/read_through.go vendored Normal file
View File

@ -0,0 +1,61 @@
// Copyright 2014 beego Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cache
import (
"context"
"time"
"github.com/beego/beego/v2/core/berror"
)
// readThroughCache is a decorator
// add the read through function to the original Cache function
type readThroughCache struct {
Cache
expiration time.Duration
loadFunc func(ctx context.Context, key string) (any, error)
}
// NewReadThroughCache create readThroughCache
func NewReadThroughCache(cache Cache, expiration time.Duration,
loadFunc func(ctx context.Context, key string) (any, error),
) (Cache, error) {
if loadFunc == nil {
return nil, berror.Error(InvalidLoadFunc, "loadFunc cannot be nil")
}
return &readThroughCache{
Cache: cache,
expiration: expiration,
loadFunc: loadFunc,
}, nil
}
// Get will try to call the LoadFunc to load data if the Cache returns value nil or non-nil error.
func (c *readThroughCache) Get(ctx context.Context, key string) (any, error) {
val, err := c.Cache.Get(ctx, key)
if val == nil || err != nil {
val, err = c.loadFunc(ctx, key)
if err != nil {
return nil, berror.Wrap(
err, LoadFuncFailed, "cache unable to load data")
}
err = c.Cache.Put(ctx, key, val, c.expiration)
if err != nil {
return val, err
}
}
return val, nil
}

144
client/cache/read_through_test.go vendored Normal file
View File

@ -0,0 +1,144 @@
// Copyright 2014 beego Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cache
import (
"context"
"errors"
"testing"
"time"
"github.com/beego/beego/v2/core/berror"
"github.com/stretchr/testify/assert"
)
func TestReadThroughCache_Memory_Get(t *testing.T) {
bm, err := NewCache("memory", `{"interval":20}`)
assert.Nil(t, err)
testReadThroughCacheGet(t, bm)
}
func TestReadThroughCache_file_Get(t *testing.T) {
fc := NewFileCache().(*FileCache)
fc.CachePath = "////aaa"
err := fc.Init()
assert.NotNil(t, err)
fc.CachePath = getTestCacheFilePath()
err = fc.Init()
assert.Nil(t, err)
testReadThroughCacheGet(t, fc)
}
func testReadThroughCacheGet(t *testing.T, bm Cache) {
testCases := []struct {
name string
key string
value string
cache Cache
wantErr error
}{
{
name: "Get load err",
key: "key0",
cache: func() Cache {
kvs := map[string]any{"key0": "value0"}
db := &MockOrm{kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
val, er := db.Load(key)
if er != nil {
return nil, er
}
return val, nil
}
c, err := NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
wantErr: func() error {
err := errors.New("the key not exist")
return berror.Wrap(
err, LoadFuncFailed, "cache unable to load data")
}(),
},
{
name: "Get cache exist",
key: "key1",
value: "value1",
cache: func() Cache {
keysMap := map[string]int{"key1": 1}
kvs := map[string]any{"key1": "value1"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
val, er := db.Load(key)
if er != nil {
return nil, er
}
return val, nil
}
c, err := NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
err = c.Put(context.Background(), "key1", "value1", 3*time.Second)
assert.Nil(t, err)
return c
}(),
},
{
name: "Get loadFunc exist",
key: "key2",
value: "value2",
cache: func() Cache {
keysMap := map[string]int{"key2": 1}
kvs := map[string]any{"key2": "value2"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
val, er := db.Load(key)
if er != nil {
return nil, er
}
return val, nil
}
c, err := NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
},
}
_, err := NewReadThroughCache(bm, 3*time.Second, nil)
assert.Equal(t, berror.Error(InvalidLoadFunc, "loadFunc cannot be nil"), err)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
c := tc.cache
val, err := c.Get(context.Background(), tc.key)
if err != nil {
assert.EqualError(t, tc.wantErr, err.Error())
return
}
assert.Equal(t, tc.value, val)
})
}
}
type MockOrm struct {
keysMap map[string]int
kvs map[string]any
}
func (m *MockOrm) Load(key string) (any, error) {
_, ok := m.keysMap[key]
if !ok {
return nil, errors.New("the key not exist")
}
return m.kvs[key], nil
}

View File

@ -16,11 +16,14 @@ package redis
import (
"context"
"errors"
"fmt"
"os"
"testing"
"time"
"github.com/beego/beego/v2/core/berror"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
@ -63,7 +66,7 @@ func TestRedisCache(t *testing.T) {
val, _ = bm.Get(context.Background(), "astaxie")
v, _ = redis.Int(val, err)
assert.Equal(t, 1, v)
bm.Delete(context.Background(), "astaxie")
assert.Nil(t, bm.Delete(context.Background(), "astaxie"))
res, _ = bm.IsExist(context.Background(), "astaxie")
assert.False(t, res)
@ -133,3 +136,116 @@ func TestCacheScan(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 0, len(keys))
}
func TestReadThroughCache_redis_Get(t *testing.T) {
bm, err := cache.NewCache("redis", fmt.Sprintf(`{"conn": "%s"}`, "127.0.0.1:6379"))
assert.Nil(t, err)
testReadThroughCacheGet(t, bm)
}
func testReadThroughCacheGet(t *testing.T, bm cache.Cache) {
testCases := []struct {
name string
key string
value string
cache cache.Cache
wantErr error
}{
{
name: "Get load err",
key: "key0",
cache: func() cache.Cache {
kvs := map[string]any{"key0": "value0"}
db := &MockOrm{kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
v, er := db.Load(key)
if er != nil {
return nil, er
}
val := []byte(v.(string))
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
wantErr: func() error {
err := errors.New("the key not exist")
return berror.Wrap(
err, cache.LoadFuncFailed, "cache unable to load data")
}(),
},
{
name: "Get cache exist",
key: "key1",
value: "value1",
cache: func() cache.Cache {
keysMap := map[string]int{"key1": 1}
kvs := map[string]any{"key1": "value1"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
v, er := db.Load(key)
if er != nil {
return nil, er
}
val := []byte(v.(string))
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
err = c.Put(context.Background(), "key1", "value1", 3*time.Second)
assert.Nil(t, err)
return c
}(),
},
{
name: "Get loadFunc exist",
key: "key2",
value: "value2",
cache: func() cache.Cache {
keysMap := map[string]int{"key2": 1}
kvs := map[string]any{"key2": "value2"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
v, er := db.Load(key)
if er != nil {
return nil, er
}
val := []byte(v.(string))
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
},
}
_, err := cache.NewReadThroughCache(bm, 3*time.Second, nil)
assert.Equal(t, berror.Error(cache.InvalidLoadFunc, "loadFunc cannot be nil"), err)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
bs := []byte(tc.value)
c := tc.cache
val, err := c.Get(context.Background(), tc.key)
if err != nil {
assert.EqualError(t, tc.wantErr, err.Error())
return
}
assert.Equal(t, bs, val)
})
}
}
type MockOrm struct {
keysMap map[string]int
kvs map[string]any
}
func (m *MockOrm) Load(key string) (any, error) {
_, ok := m.keysMap[key]
if !ok {
return nil, errors.New("the key not exist")
}
return m.kvs[key], nil
}

View File

@ -2,6 +2,7 @@ package ssdb
import (
"context"
"errors"
"fmt"
"os"
"strconv"
@ -9,6 +10,8 @@ import (
"testing"
"time"
"github.com/beego/beego/v2/core/berror"
"github.com/stretchr/testify/assert"
"github.com/beego/beego/v2/client/cache"
@ -98,3 +101,112 @@ func TestSsdbcacheCache(t *testing.T) {
assert.False(t, e1)
assert.False(t, e2)
}
func TestReadThroughCache_ssdb_Get(t *testing.T) {
bm, err := cache.NewCache("ssdb", fmt.Sprintf(`{"conn": "%s"}`, "127.0.0.1:8888"))
assert.Nil(t, err)
testReadThroughCacheGet(t, bm)
}
func testReadThroughCacheGet(t *testing.T, bm cache.Cache) {
testCases := []struct {
name string
key string
value string
cache cache.Cache
wantErr error
}{
{
name: "Get load err",
key: "key0",
cache: func() cache.Cache {
kvs := map[string]any{"key0": "value0"}
db := &MockOrm{kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
val, er := db.Load(key)
if er != nil {
return nil, er
}
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
wantErr: func() error {
err := errors.New("the key not exist")
return berror.Wrap(
err, cache.LoadFuncFailed, "cache unable to load data")
}(),
},
{
name: "Get cache exist",
key: "key1",
value: "value1",
cache: func() cache.Cache {
keysMap := map[string]int{"key1": 1}
kvs := map[string]any{"key1": "value1"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
val, er := db.Load(key)
if er != nil {
return nil, er
}
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
err = c.Put(context.Background(), "key1", "value1", 3*time.Second)
assert.Nil(t, err)
return c
}(),
},
{
name: "Get loadFunc exist",
key: "key2",
value: "value2",
cache: func() cache.Cache {
keysMap := map[string]int{"key2": 1}
kvs := map[string]any{"key2": "value2"}
db := &MockOrm{keysMap: keysMap, kvs: kvs}
loadfunc := func(ctx context.Context, key string) (any, error) {
val, er := db.Load(key)
if er != nil {
return nil, er
}
return val, nil
}
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
assert.Nil(t, err)
return c
}(),
},
}
_, err := cache.NewReadThroughCache(bm, 3*time.Second, nil)
assert.Equal(t, berror.Error(cache.InvalidLoadFunc, "loadFunc cannot be nil"), err)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
c := tc.cache
val, err := c.Get(context.Background(), tc.key)
if err != nil {
assert.EqualError(t, tc.wantErr, err.Error())
return
}
assert.Equal(t, tc.value, val)
})
}
}
type MockOrm struct {
keysMap map[string]int
kvs map[string]any
}
func (m *MockOrm) Load(key string) (any, error) {
_, ok := m.keysMap[key]
if !ok {
return nil, errors.New("the key not exist")
}
return m.kvs[key], nil
}