From 9ced4fdd6d268d4a4a23b5ab7881b6dbdd66a954 Mon Sep 17 00:00:00 2001 From: Ming Deng Date: Wed, 15 Sep 2021 11:12:57 +0800 Subject: [PATCH 1/5] Update change log --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5201e3a..e5bdf27d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# developing +# v2.0.2-beta.1 - Add a custom option for whether to escape HTML special characters when processing http request parameters. [4701](https://github.com/beego/beego/pull/4701) - Always set the response status in the CustomAbort function. [4686](https://github.com/beego/beego/pull/4686) From 3981234bfbcc6a71b6ba6c3b302b631efba87cde Mon Sep 17 00:00:00 2001 From: Deng Ming Date: Tue, 19 Oct 2021 21:47:09 +0800 Subject: [PATCH 2/5] set default rate and capacity for ratelimit filter --- CHANGELOG.md | 1 + server/web/filter/ratelimit/limiter.go | 37 ++++++++----------- server/web/filter/ratelimit/token_bucket.go | 4 +- .../web/filter/ratelimit/token_bucket_test.go | 8 ++-- server/web/filter/session/filter.go | 1 + 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5bdf27d..18231a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ - Fix 4698: Prompt error when config format is incorrect. [4757](https://github.com/beego/beego/pull/4757) - Fix 4674: Tx Orm missing debug log [4756](https://github.com/beego/beego/pull/4756) - Fix 4759: fix numeric notation of permissions [4759](https://github.com/beego/beego/pull/4759) +- set default rate and capacity for ratelimit filter [4796](https://github.com/beego/beego/pull/4796) ## Fix Sonar - [4677](https://github.com/beego/beego/pull/4677) diff --git a/server/web/filter/ratelimit/limiter.go b/server/web/filter/ratelimit/limiter.go index 5b64b5dd..e0aac5c1 100644 --- a/server/web/filter/ratelimit/limiter.go +++ b/server/web/filter/ratelimit/limiter.go @@ -15,7 +15,6 @@ package ratelimit import ( - "net/http" "sync" "time" @@ -23,11 +22,6 @@ import ( "github.com/beego/beego/v2/server/web/context" ) -// Limiter is an interface used to ratelimit -type Limiter interface { - take(amount uint, r *http.Request) bool -} - // limiterOption is constructor option type limiterOption func(l *limiter) @@ -37,7 +31,7 @@ type limiter struct { rate time.Duration buckets map[string]bucket bucketFactory func(opts ...bucketOption) bucket - sessionKey func(r *http.Request) string + sessionKey func(ctx *context.Context) string resp RejectionResponse } @@ -60,10 +54,10 @@ var defaultRejectionResponse = RejectionResponse{ func NewLimiter(opts ...limiterOption) web.FilterFunc { l := &limiter{ buckets: make(map[string]bucket), - sessionKey: func(r *http.Request) string { - return defaultSessionKey(r) - }, - bucketFactory: NewTokenBucket, + sessionKey: defaultSessionKey, + rate: time.Millisecond * 10, + capacity: 100, + bucketFactory: newTokenBucket, resp: defaultRejectionResponse, } for _, o := range opts { @@ -71,7 +65,7 @@ func NewLimiter(opts ...limiterOption) web.FilterFunc { } return func(ctx *context.Context) { - if !l.take(perRequestConsumedAmount, ctx.Request) { + if !l.take(perRequestConsumedAmount, ctx) { ctx.ResponseWriter.WriteHeader(l.resp.code) ctx.WriteString(l.resp.body) } @@ -79,8 +73,8 @@ func NewLimiter(opts ...limiterOption) web.FilterFunc { } // WithSessionKey return limiterOption. WithSessionKey config func -// which defines the request characteristic againstthe limit is applied -func WithSessionKey(f func(r *http.Request) string) limiterOption { +// which defines the request characteristic against the limit is applied +func WithSessionKey(f func(ctx *context.Context) string) limiterOption { return func(l *limiter) { l.sessionKey = f } @@ -119,16 +113,16 @@ func WithRejectionResponse(resp RejectionResponse) limiterOption { } } -func (l *limiter) take(amount uint, r *http.Request) bool { - bucket := l.getBucket(r) +func (l *limiter) take(amount uint, ctx *context.Context) bool { + bucket := l.getBucket(ctx) if bucket == nil { return true } return bucket.take(amount) } -func (l *limiter) getBucket(r *http.Request) bucket { - key := l.sessionKey(r) +func (l *limiter) getBucket(ctx *context.Context) bucket { + key := l.sessionKey(ctx) l.RLock() b, ok := l.buckets[key] l.RUnlock() @@ -152,11 +146,12 @@ func (l *limiter) createBucket(key string) bucket { return b } -func defaultSessionKey(r *http.Request) string { - return "" +func defaultSessionKey(ctx *context.Context) string { + return "BEEGO_ALL" } -func RemoteIPSessionKey(r *http.Request) string { +func RemoteIPSessionKey(ctx *context.Context) string { + r := ctx.Request IPAddress := r.Header.Get("X-Real-Ip") if IPAddress == "" { IPAddress = r.Header.Get("X-Forwarded-For") diff --git a/server/web/filter/ratelimit/token_bucket.go b/server/web/filter/ratelimit/token_bucket.go index 5906ee9e..da7bb7fc 100644 --- a/server/web/filter/ratelimit/token_bucket.go +++ b/server/web/filter/ratelimit/token_bucket.go @@ -13,8 +13,8 @@ type tokenBucket struct { rate time.Duration } -// NewTokenBucket return an bucket that implements token bucket -func NewTokenBucket(opts ...bucketOption) bucket { +// newTokenBucket return an bucket that implements token bucket +func newTokenBucket(opts ...bucketOption) bucket { b := &tokenBucket{lastCheckAt: time.Now()} for _, o := range opts { o(b) diff --git a/server/web/filter/ratelimit/token_bucket_test.go b/server/web/filter/ratelimit/token_bucket_test.go index 93a1b3bd..91088eb6 100644 --- a/server/web/filter/ratelimit/token_bucket_test.go +++ b/server/web/filter/ratelimit/token_bucket_test.go @@ -8,24 +8,24 @@ import ( ) func TestGetRate(t *testing.T) { - b := NewTokenBucket(withRate(1 * time.Second)).(*tokenBucket) + b := newTokenBucket(withRate(1 * time.Second)).(*tokenBucket) assert.Equal(t, b.getRate(), 1*time.Second) } func TestGetRemainingAndCapacity(t *testing.T) { - b := NewTokenBucket(withCapacity(10)) + b := newTokenBucket(withCapacity(10)) assert.Equal(t, b.getRemaining(), uint(10)) assert.Equal(t, b.getCapacity(), uint(10)) } func TestTake(t *testing.T) { - b := NewTokenBucket(withCapacity(10), withRate(10*time.Millisecond)).(*tokenBucket) + b := newTokenBucket(withCapacity(10), withRate(10*time.Millisecond)).(*tokenBucket) for i := 0; i < 10; i++ { assert.True(t, b.take(1)) } assert.False(t, b.take(1)) assert.Equal(t, b.getRemaining(), uint(0)) - b = NewTokenBucket(withCapacity(1), withRate(1*time.Millisecond)).(*tokenBucket) + b = newTokenBucket(withCapacity(1), withRate(1*time.Millisecond)).(*tokenBucket) assert.True(t, b.take(1)) time.Sleep(2 * time.Millisecond) assert.True(t, b.take(1)) diff --git a/server/web/filter/session/filter.go b/server/web/filter/session/filter.go index 40f3e198..b3bb99c5 100644 --- a/server/web/filter/session/filter.go +++ b/server/web/filter/session/filter.go @@ -11,6 +11,7 @@ import ( // Session maintain session for web service // Session new a session storage and store it into webContext.Context +// experimental feature, we may change this in the future func Session(providerType session.ProviderType, options ...session.ManagerConfigOpt) web.FilterChain { sessionConfig := session.NewManagerConfig(options...) sessionManager, _ := session.NewManager(string(providerType), sessionConfig) From 0347cfc7a59e4c33932c4dc980be0e4ce38b1dce Mon Sep 17 00:00:00 2001 From: Deng Ming Date: Wed, 20 Oct 2021 20:39:04 +0800 Subject: [PATCH 3/5] fxi4782: must set status before rendering error page --- CHANGELOG.md | 2 ++ server/web/config.go | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18231a90..ddac259a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,8 @@ - Fix 4674: Tx Orm missing debug log [4756](https://github.com/beego/beego/pull/4756) - Fix 4759: fix numeric notation of permissions [4759](https://github.com/beego/beego/pull/4759) - set default rate and capacity for ratelimit filter [4796](https://github.com/beego/beego/pull/4796) +- Fix 4782: must set status before rendering error page [4797](https://github.com/beego/beego/pull/4797) + ## Fix Sonar - [4677](https://github.com/beego/beego/pull/4677) diff --git a/server/web/config.go b/server/web/config.go index 6bddfe4a..006201a6 100644 --- a/server/web/config.go +++ b/server/web/config.go @@ -507,14 +507,16 @@ func defaultRecoverPanic(ctx *context.Context, cfg *Config) { logs.Critical(fmt.Sprintf("%s:%d", file, line)) stack = stack + fmt.Sprintln(fmt.Sprintf("%s:%d", file, line)) } - if cfg.RunMode == DEV && cfg.EnableErrorsRender { - showErr(err, ctx, stack) - } + if ctx.Output.Status != 0 { ctx.ResponseWriter.WriteHeader(ctx.Output.Status) } else { ctx.ResponseWriter.WriteHeader(500) } + + if cfg.RunMode == DEV && cfg.EnableErrorsRender { + showErr(err, ctx, stack) + } } } From fe4b84e4f6965956badafaedb4a56dbfdc933897 Mon Sep 17 00:00:00 2001 From: Deng Ming Date: Mon, 25 Oct 2021 22:22:09 +0800 Subject: [PATCH 4/5] fix4791: delay to format parameter --- CHANGELOG.md | 1 + core/logs/formatter.go | 2 +- core/logs/log.go | 26 +++++++++++++------------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddac259a..e7c26c42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ - Fix 4759: fix numeric notation of permissions [4759](https://github.com/beego/beego/pull/4759) - set default rate and capacity for ratelimit filter [4796](https://github.com/beego/beego/pull/4796) - Fix 4782: must set status before rendering error page [4797](https://github.com/beego/beego/pull/4797) +- Fix 4791: delay to format parameter in log module [4804](https://github.com/beego/beego/pull/4804) ## Fix Sonar diff --git a/core/logs/formatter.go b/core/logs/formatter.go index 80b30fa0..6e9a5bd8 100644 --- a/core/logs/formatter.go +++ b/core/logs/formatter.go @@ -60,7 +60,7 @@ func GetFormatter(name string) (LogFormatter, bool) { return res, ok } -// 'w' when, 'm' msg,'f' filename，'F' full path，'n' line number +// ToString 'w' when, 'm' msg,'f' filename，'F' full path，'n' line number // 'l' level number, 't' prefix of level type, 'T' full name of level type func (p *PatternLogFormatter) ToString(lm *LogMsg) string { s := []rune(p.Pattern) diff --git a/core/logs/log.go b/core/logs/log.go index ad2ef953..9ce8a07b 100644 --- a/core/logs/log.go +++ b/core/logs/log.go @@ -705,61 +705,61 @@ func SetLogger(adapter string, config ...string) error { // Emergency logs a message at emergency level. func Emergency(f interface{}, v ...interface{}) { - beeLogger.Emergency(formatLog(f, v...)) + beeLogger.Emergency(formatPattern(f, v...), v...) } // Alert logs a message at alert level. func Alert(f interface{}, v ...interface{}) { - beeLogger.Alert(formatLog(f, v...)) + beeLogger.Alert(formatPattern(f, v...), v...) } // Critical logs a message at critical level. func Critical(f interface{}, v ...interface{}) { - beeLogger.Critical(formatLog(f, v...)) + beeLogger.Critical(formatPattern(f, v...), v...) } // Error logs a message at error level. func Error(f interface{}, v ...interface{}) { - beeLogger.Error(formatLog(f, v...)) + beeLogger.Error(formatPattern(f, v...), v...) } // Warning logs a message at warning level. func Warning(f interface{}, v ...interface{}) { - beeLogger.Warn(formatLog(f, v...)) + beeLogger.Warn(formatPattern(f, v...), v...) } // Warn compatibility alias for Warning() func Warn(f interface{}, v ...interface{}) { - beeLogger.Warn(formatLog(f, v...)) + beeLogger.Warn(formatPattern(f, v...), v...) } // Notice logs a message at notice level. func Notice(f interface{}, v ...interface{}) { - beeLogger.Notice(formatLog(f, v...)) + beeLogger.Notice(formatPattern(f, v...), v...) } // Informational logs a message at info level. func Informational(f interface{}, v ...interface{}) { - beeLogger.Info(formatLog(f, v...)) + beeLogger.Info(formatPattern(f, v...), v...) } // Info compatibility alias for Warning() func Info(f interface{}, v ...interface{}) { - beeLogger.Info(formatLog(f, v...)) + beeLogger.Info(formatPattern(f, v...), v...) } // Debug logs a message at debug level. func Debug(f interface{}, v ...interface{}) { - beeLogger.Debug(formatLog(f, v...)) + beeLogger.Debug(formatPattern(f, v...), v...) } // Trace logs a message at trace level. // compatibility alias for Warning() func Trace(f interface{}, v ...interface{}) { - beeLogger.Trace(formatLog(f, v...)) + beeLogger.Trace(formatPattern(f, v...), v...) } -func formatLog(f interface{}, v ...interface{}) string { +func formatPattern(f interface{}, v ...interface{}) string { var msg string switch f.(type) { case string: @@ -778,5 +778,5 @@ func formatLog(f interface{}, v ...interface{}) string { } msg += strings.Repeat(" %v", len(v)) } - return fmt.Sprintf(msg, v...) + return msg } From b8cf07fa18f665f2c48b2a6d4e5cedde53a28a1a Mon Sep 17 00:00:00 2001 From: Deng Ming Date: Sun, 9 Jan 2022 16:49:11 +0800 Subject: [PATCH 5/5] fix bug: etcd should use etcd as adapter name --- CHANGELOG.md | 2 ++ core/config/etcd/config.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7c26c42..72eb9904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +# developing +- [https://github.com/beego/beego/pull/4845](https://github.com/beego/beego/pull/4845) # v2.0.2-beta.1 - Add a custom option for whether to escape HTML special characters when processing http request parameters. [4701](https://github.com/beego/beego/pull/4701) diff --git a/core/config/etcd/config.go b/core/config/etcd/config.go index fb12b065..e0e30b44 100644 --- a/core/config/etcd/config.go +++ b/core/config/etcd/config.go @@ -46,7 +46,7 @@ func newEtcdConfiger(client *clientv3.Client, prefix string) *EtcdConfiger { return res } -// reader is an general implementation that read config from etcd. +// reader is a general implementation that read config from etcd. func (e *EtcdConfiger) reader(ctx context.Context, key string) (string, error) { resp, err := get(e.client, e.prefix+key) if err != nil { @@ -188,5 +188,5 @@ func get(client *clientv3.Client, key string) (*clientv3.GetResponse, error) { } func init() { - config.Register("json", &EtcdConfigerProvider{}) + config.Register("etcd", &EtcdConfigerProvider{}) }