* feature extend readthrough for cache module (#5116) * feature 增加readthrough * feature: add write though for cache mode (#5117) * feature: add writethough for cache mode * feature add singleflight cache (#5119) * build(deps): bump go.opentelemetry.io/otel/trace from 1.8.0 to 1.11.2 Bumps [go.opentelemetry.io/otel/trace](https://github.com/open-telemetry/opentelemetry-go) from 1.8.0 to 1.11.2. - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.8.0...v1.11.2) --- updated-dependencies: - dependency-name: go.opentelemetry.io/otel/trace dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * fix 5129: must set formatter after init the logger * remove beego.vip * build(deps): bump actions/stale from 5 to 7 Bumps [actions/stale](https://github.com/actions/stale) from 5 to 7. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5...v7) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> * fix 5079: only log msg when the channel is not closed (#5132) * optimize test * upgrade otel dependencies to v1.11.2 * format code * Bloom filter cache (#5126) * feature: add bloom filter cache * feature upload remove all temp file * bugfix Controller SaveToFile remove all temp file * rft: motify BeeLogger signalChan (#5139) * add non-block write log in asynchronous mode (#5150) * add non-block write log in asynchronous mode --------- Co-authored-by: chenhaokun <chenhaokun@itiger.com> * fix the docsite URL (#5173) * Unified gopkg.in/yaml version to v2 (#5169) * Unified gopkg.in/yaml version to v2 and go mod tidy * update CHANGELOG * bugfix: protect field access with lock to avoid possible data race (#5211) * fix some comments (#5194) Signed-off-by: cui fliter <imcusg@gmail.com> * build(deps): bump github.com/prometheus/client_golang (#5213) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.14.0 to 1.15.1. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.14.0...v1.15.1) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump go.etcd.io/etcd/client/v3 from 3.5.4 to 3.5.9 (#5209) Bumps [go.etcd.io/etcd/client/v3](https://github.com/etcd-io/etcd) from 3.5.4 to 3.5.9. - [Release notes](https://github.com/etcd-io/etcd/releases) - [Commits](https://github.com/etcd-io/etcd/compare/v3.5.4...v3.5.9) --- updated-dependencies: - dependency-name: go.etcd.io/etcd/client/v3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * cache: fix typo and optimize the naming * Release 2.1.0 change log * bugfix: beegoAppConfig String and Strings function has bug * httplib: fix unstable test, do not use httplib.org * chore: pkg imported more than once * chore: fmt modify * chore: Use github.com/go-kit/log * chore: unnecessary use of fmt.Sprintf * fix: golangci-lint error * orm: refactor ORM introducing internal/models pkg * remove adapter package * build(deps): bump github.com/bits-and-blooms/bloom/v3 Bumps [github.com/bits-and-blooms/bloom/v3](https://github.com/bits-and-blooms/bloom) from 3.3.1 to 3.5.0. - [Release notes](https://github.com/bits-and-blooms/bloom/releases) - [Commits](https://github.com/bits-and-blooms/bloom/compare/v3.3.1...v3.5.0) --- updated-dependencies: - dependency-name: github.com/bits-and-blooms/bloom/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * feat: add write-delete cache mode * fix: unnecessary assignment to the blank identifier * fix: add change into .CHANGELOG file * build(deps): bump golang.org/x/sync from 0.1.0 to 0.3.0 Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.1.0 to 0.3.0. - [Commits](https://github.com/golang/sync/compare/v0.1.0...v0.3.0) --- updated-dependencies: - dependency-name: golang.org/x/sync dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump golang.org/x/crypto Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.0.0-20220315160706-3147a52a75dd to 0.10.0. - [Commits](https://github.com/golang/crypto/commits/v0.10.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * remove golang--lint-ci * Beego web.Run() runs the server twice * fix 5255: Check the rows.Err() if rows.Next() is false * closes 5254: %COL% should be a common placeholder * build(deps): bump github.com/prometheus/client_golang Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.15.1 to 1.16.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.15.1...v1.16.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * fix: use of ioutil package (#5261) * fix ioutil.NopCloser * fix ioutil.ReadAll * fix ioutil.ReadFile * fix ioutil.WriteFile * run goimports -w -format-only ./ * update CHANGELOG.md * feature: add write-double-delete cache mode (#5263) * cache/redis: support skipEmptyPrefix option (#5264) * fix: refactor InsertValue method (#5267) * fix: refactor insertValue method and add the test * fix: exec goimports and add Licence file header * fix: modify construct method of dbBase * fix: add modify record into CHANGELOG * fix: modify InsertOrUpdate method (#5269) * fix: modify InsertOrUpdate method, Remove the isMulti variable and its associated code * fix: Delete unnecessary judgment branches * fix: add modify record into CHANGELOG * cache/redis: use redisConfig to receive incoming JSON (previously using a map) (#5268) * refactor cache/redis: Use redisConfig to receive incoming JSON (previously using a map). * refactor cache/redis: Use the string type to receive JSON parameters. --------- Co-authored-by: Tan <tanqianheng@gmail.com> * fix: refactor Delete method (#5271) * fix: refactor Delete method and add test * fix: add modify record into CHANGELOG * fix: refactor update sql (#5274) * fix: refactor UpdateSQL method and add test * fix: add modify record into CHANGELOG * fix: modify url in the CHANGELOG * fix: modify pr url in the CHANGELOG * Fix setPK function for table without primary key (#5276) --------- Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: cui fliter <imcusg@gmail.com> Co-authored-by: Stone-afk <73482944+Stone-afk@users.noreply.github.com> Co-authored-by: hookokoko <hooko@tju.edu.cn> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: hookokoko <648646891@qq.com> Co-authored-by: Stone-afk <1711865140@qq.com> Co-authored-by: chenhaokun <chenhaokun@itiger.com> Co-authored-by: Xuing <admin@xuing.cn> Co-authored-by: cui fliter <imcusg@gmail.com> Co-authored-by: guoguangwu <guoguangwu@magic-shield.com> Co-authored-by: uzziah <uzziahlin@gmail.com> Co-authored-by: Hanjiang Yu <delacroix.yu@gmail.com> Co-authored-by: Kota <mdryzk64smsh@gmail.com> Co-authored-by: Uzziah <120019273+uzziahlin@users.noreply.github.com> Co-authored-by: Handkerchiefs-t <59816423+Handkerchiefs-t@users.noreply.github.com> Co-authored-by: Tan <tanqianheng@gmail.com> Co-authored-by: mlgd <mlgd17@gmail.com>
362 lines
8.7 KiB
Go
362 lines
8.7 KiB
Go
// Copyright 2014 beego Author. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package redis
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/gomodule/redigo/redis"
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/beego/beego/v2/client/cache"
|
|
"github.com/beego/beego/v2/core/berror"
|
|
)
|
|
|
|
func TestRedisCache(t *testing.T) {
|
|
redisAddr := os.Getenv("REDIS_ADDR")
|
|
if redisAddr == "" {
|
|
redisAddr = "127.0.0.1:6379"
|
|
}
|
|
|
|
bm, err := cache.NewCache("redis", fmt.Sprintf(`{"conn": "%s"}`, redisAddr))
|
|
assert.Nil(t, err)
|
|
timeoutDuration := 3 * time.Second
|
|
|
|
assert.Nil(t, bm.Put(context.Background(), "astaxie", 1, timeoutDuration))
|
|
|
|
res, _ := bm.IsExist(context.Background(), "astaxie")
|
|
assert.True(t, res)
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
res, _ = bm.IsExist(context.Background(), "astaxie")
|
|
assert.False(t, res)
|
|
|
|
assert.Nil(t, bm.Put(context.Background(), "astaxie", 1, timeoutDuration))
|
|
|
|
val, _ := bm.Get(context.Background(), "astaxie")
|
|
v, _ := redis.Int(val, err)
|
|
assert.Equal(t, 1, v)
|
|
|
|
assert.Nil(t, bm.Incr(context.Background(), "astaxie"))
|
|
val, _ = bm.Get(context.Background(), "astaxie")
|
|
v, _ = redis.Int(val, err)
|
|
assert.Equal(t, 2, v)
|
|
|
|
assert.Nil(t, bm.Decr(context.Background(), "astaxie"))
|
|
|
|
val, _ = bm.Get(context.Background(), "astaxie")
|
|
v, _ = redis.Int(val, err)
|
|
assert.Equal(t, 1, v)
|
|
assert.Nil(t, bm.Delete(context.Background(), "astaxie"))
|
|
|
|
res, _ = bm.IsExist(context.Background(), "astaxie")
|
|
assert.False(t, res)
|
|
|
|
assert.Nil(t, bm.Put(context.Background(), "astaxie", "author", timeoutDuration))
|
|
// test string
|
|
|
|
res, _ = bm.IsExist(context.Background(), "astaxie")
|
|
assert.True(t, res)
|
|
|
|
val, _ = bm.Get(context.Background(), "astaxie")
|
|
vs, _ := redis.String(val, err)
|
|
assert.Equal(t, "author", vs)
|
|
|
|
// test GetMulti
|
|
assert.Nil(t, bm.Put(context.Background(), "astaxie1", "author1", timeoutDuration))
|
|
|
|
res, _ = bm.IsExist(context.Background(), "astaxie1")
|
|
assert.True(t, res)
|
|
|
|
vv, _ := bm.GetMulti(context.Background(), []string{"astaxie", "astaxie1"})
|
|
assert.Equal(t, 2, len(vv))
|
|
vs, _ = redis.String(vv[0], nil)
|
|
assert.Equal(t, "author", vs)
|
|
|
|
vs, _ = redis.String(vv[1], nil)
|
|
assert.Equal(t, "author1", vs)
|
|
|
|
vv, _ = bm.GetMulti(context.Background(), []string{"astaxie0", "astaxie1"})
|
|
assert.Nil(t, vv[0])
|
|
|
|
vs, _ = redis.String(vv[1], nil)
|
|
assert.Equal(t, "author1", vs)
|
|
|
|
// test clear all
|
|
assert.Nil(t, bm.ClearAll(context.Background()))
|
|
}
|
|
|
|
func TestCacheScan(t *testing.T) {
|
|
timeoutDuration := 10 * time.Second
|
|
|
|
addr := os.Getenv("REDIS_ADDR")
|
|
if addr == "" {
|
|
addr = "127.0.0.1:6379"
|
|
}
|
|
|
|
// init
|
|
bm, err := cache.NewCache("redis", fmt.Sprintf(`{"conn": "%s"}`, addr))
|
|
|
|
assert.Nil(t, err)
|
|
// insert all
|
|
for i := 0; i < 100; i++ {
|
|
assert.Nil(t, bm.Put(context.Background(), fmt.Sprintf("astaxie%d", i), fmt.Sprintf("author%d", i), timeoutDuration))
|
|
}
|
|
time.Sleep(time.Second)
|
|
// scan all for the first time
|
|
keys, err := bm.(*Cache).Scan(DefaultKey + ":*")
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, 100, len(keys), "scan all error")
|
|
|
|
// clear all
|
|
assert.Nil(t, bm.ClearAll(context.Background()))
|
|
|
|
// scan all for the second time
|
|
keys, err = bm.(*Cache).Scan(DefaultKey + ":*")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, 0, len(keys))
|
|
}
|
|
|
|
func TestReadThroughCache_redis_Get(t *testing.T) {
|
|
bm, err := cache.NewCache("redis", fmt.Sprintf(`{"conn": "%s"}`, "127.0.0.1:6379"))
|
|
assert.Nil(t, err)
|
|
|
|
testReadThroughCacheGet(t, bm)
|
|
}
|
|
|
|
func testReadThroughCacheGet(t *testing.T, bm cache.Cache) {
|
|
testCases := []struct {
|
|
name string
|
|
key string
|
|
value string
|
|
cache cache.Cache
|
|
wantErr error
|
|
}{
|
|
{
|
|
name: "Get load err",
|
|
key: "key0",
|
|
cache: func() cache.Cache {
|
|
kvs := map[string]any{"key0": "value0"}
|
|
db := &MockOrm{kvs: kvs}
|
|
loadfunc := func(ctx context.Context, key string) (any, error) {
|
|
v, er := db.Load(key)
|
|
if er != nil {
|
|
return nil, er
|
|
}
|
|
val := []byte(v.(string))
|
|
return val, nil
|
|
}
|
|
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
|
|
assert.Nil(t, err)
|
|
return c
|
|
}(),
|
|
wantErr: func() error {
|
|
err := errors.New("the key not exist")
|
|
return berror.Wrap(
|
|
err, cache.LoadFuncFailed, "cache unable to load data")
|
|
}(),
|
|
},
|
|
{
|
|
name: "Get cache exist",
|
|
key: "key1",
|
|
value: "value1",
|
|
cache: func() cache.Cache {
|
|
keysMap := map[string]int{"key1": 1}
|
|
kvs := map[string]any{"key1": "value1"}
|
|
db := &MockOrm{keysMap: keysMap, kvs: kvs}
|
|
loadfunc := func(ctx context.Context, key string) (any, error) {
|
|
v, er := db.Load(key)
|
|
if er != nil {
|
|
return nil, er
|
|
}
|
|
val := []byte(v.(string))
|
|
return val, nil
|
|
}
|
|
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
|
|
assert.Nil(t, err)
|
|
err = c.Put(context.Background(), "key1", "value1", 3*time.Second)
|
|
assert.Nil(t, err)
|
|
return c
|
|
}(),
|
|
},
|
|
{
|
|
name: "Get loadFunc exist",
|
|
key: "key2",
|
|
value: "value2",
|
|
cache: func() cache.Cache {
|
|
keysMap := map[string]int{"key2": 1}
|
|
kvs := map[string]any{"key2": "value2"}
|
|
db := &MockOrm{keysMap: keysMap, kvs: kvs}
|
|
loadfunc := func(ctx context.Context, key string) (any, error) {
|
|
v, er := db.Load(key)
|
|
if er != nil {
|
|
return nil, er
|
|
}
|
|
val := []byte(v.(string))
|
|
return val, nil
|
|
}
|
|
c, err := cache.NewReadThroughCache(bm, 3*time.Second, loadfunc)
|
|
assert.Nil(t, err)
|
|
return c
|
|
}(),
|
|
},
|
|
}
|
|
_, err := cache.NewReadThroughCache(bm, 3*time.Second, nil)
|
|
assert.Equal(t, berror.Error(cache.InvalidLoadFunc, "loadFunc cannot be nil"), err)
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
bs := []byte(tc.value)
|
|
c := tc.cache
|
|
val, err := c.Get(context.Background(), tc.key)
|
|
if err != nil {
|
|
assert.EqualError(t, tc.wantErr, err.Error())
|
|
return
|
|
}
|
|
assert.Equal(t, bs, val)
|
|
})
|
|
}
|
|
}
|
|
|
|
type MockOrm struct {
|
|
keysMap map[string]int
|
|
kvs map[string]any
|
|
}
|
|
|
|
func (m *MockOrm) Load(key string) (any, error) {
|
|
_, ok := m.keysMap[key]
|
|
if !ok {
|
|
return nil, errors.New("the key not exist")
|
|
}
|
|
return m.kvs[key], nil
|
|
}
|
|
|
|
func TestCache_associate(t *testing.T) {
|
|
testCases := []struct {
|
|
name string
|
|
skipEmptyPrefix bool
|
|
prefix string
|
|
input string
|
|
wantRes string
|
|
}{
|
|
{
|
|
name: "skip prefix",
|
|
skipEmptyPrefix: true,
|
|
prefix: "",
|
|
input: "my-key",
|
|
wantRes: "my-key",
|
|
},
|
|
{
|
|
name: "skip prefix but prefix not empty",
|
|
skipEmptyPrefix: true,
|
|
prefix: "abc",
|
|
input: "my-key",
|
|
wantRes: "abc:my-key",
|
|
},
|
|
{
|
|
name: "using empty prefix",
|
|
skipEmptyPrefix: false,
|
|
prefix: "",
|
|
input: "my-key",
|
|
wantRes: ":my-key",
|
|
},
|
|
{
|
|
name: "using prefix",
|
|
prefix: "abc",
|
|
input: "my-key",
|
|
wantRes: "abc:my-key",
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
c := NewRedisCache().(*Cache)
|
|
c.skipEmptyPrefix = tc.skipEmptyPrefix
|
|
c.key = tc.prefix
|
|
res := c.associate(tc.input)
|
|
assert.Equal(t, tc.wantRes, res)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCache_parseConf(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
|
|
configStr string
|
|
|
|
wantCache Cache
|
|
wantErr error
|
|
}{
|
|
{
|
|
name: "just conn",
|
|
configStr: `{
|
|
"conn": "127.0.0.1:6379"
|
|
}`,
|
|
|
|
wantCache: Cache{
|
|
conninfo: "127.0.0.1:6379",
|
|
dbNum: 0,
|
|
key: DefaultKey,
|
|
password: "",
|
|
maxIdle: defaultMaxIdle,
|
|
skipEmptyPrefix: false,
|
|
timeout: defaultTimeout,
|
|
},
|
|
wantErr: nil,
|
|
},
|
|
|
|
{
|
|
name: "all",
|
|
configStr: `{
|
|
"dbNum": "2",
|
|
"skipEmptyPrefix": "true",
|
|
"key": "mykey",
|
|
"conn": "redis://mypwd@127.0.0.1:6379",
|
|
"maxIdle": "10",
|
|
"timeout": "30s"
|
|
}`,
|
|
|
|
wantCache: Cache{
|
|
conninfo: "127.0.0.1:6379",
|
|
dbNum: 2,
|
|
key: "mykey",
|
|
password: "mypwd",
|
|
maxIdle: 10,
|
|
skipEmptyPrefix: true,
|
|
timeout: time.Second * 30,
|
|
},
|
|
wantErr: nil,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
c := Cache{}
|
|
err := c.parseConf(tt.configStr)
|
|
assert.Equal(t, tt.wantErr, err)
|
|
if err != nil {
|
|
return
|
|
}
|
|
assert.Equal(t, tt.wantCache, c)
|
|
})
|
|
}
|
|
}
|