Merge pull request #4446 from flycash/master

Fix 4444: Remove duration label, 404 panic, init observer once

CHANGELOG.md

@@ -0,0 +1,6 @@
+# developing
+- Remove `duration` from prometheus labels. [4391](https://github.com/beego/beego/pull/4391)
+- Fix `unknown escape sequence` in generated code. [4385](https://github.com/beego/beego/pull/4385)
+- Using fixed name `commentRouter.go` as generated file name. [4385](https://github.com/beego/beego/pull/4385)
+- Fix 4383: ORM Adapter produces panic when using orm.RegisterModelWithPrefix. [4386](https://github.com/beego/beego/pull/4386)
+- Fix 4444: panic when 404 not found. [4446](https://github.com/beego/beego/pull/4446)

adapter/metric/prometheus.go

@@ -38,7 +38,7 @@ func PrometheusMiddleWare(next http.Handler) http.Handler {
 			"appname": web.BConfig.AppName,
 		},
 		Help: "The statics info for http request",
-	}, []string{"pattern", "method", "status", "duration"})
+	}, []string{"pattern", "method", "status"})
 
 	prometheus.MustRegister(summaryVec)
 
@@ -96,5 +96,5 @@ func report(dur time.Duration, writer http.ResponseWriter, q *http.Request, vec
 		logs.Warn("we can not find the router info for this request, so request will be recorded as UNKNOWN: " + q.URL.String())
 	}
 	ms := dur / time.Millisecond
-	vec.WithLabelValues(ptn, q.Method, strconv.Itoa(status), strconv.Itoa(int(ms))).Observe(float64(ms))
+	vec.WithLabelValues(ptn, q.Method, strconv.Itoa(status)).Observe(float64(ms))
 }

adapter/metric/prometheus_test.go

@@ -35,7 +35,7 @@ func TestPrometheusMiddleWare(t *testing.T) {
 		},
 		Method: "POST",
 	}
-	vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{}, []string{"pattern", "method", "status", "duration"})
+	vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{}, []string{"pattern", "method", "status"})
 
 	report(time.Second, writer, request, vec)
 	middleware.ServeHTTP(writer, request)

client/httplib/filter/prometheus/filter.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -26,24 +27,30 @@ import (
 )
 
 type FilterChainBuilder struct {
-	summaryVec prometheus.ObserverVec
 	AppName    string
 	ServerName string
 	RunMode    string
 }
 
+var summaryVec prometheus.ObserverVec
+var initSummaryVec sync.Once
+
 func (builder *FilterChainBuilder) FilterChain(next httplib.Filter) httplib.Filter {
 
-	builder.summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+	initSummaryVec.Do(func() {
-		Name:      "beego",
+		summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Subsystem: "remote_http_request",
+			Name:      "beego",
-		ConstLabels: map[string]string{
+			Subsystem: "remote_http_request",
-			"server":  builder.ServerName,
+			ConstLabels: map[string]string{
-			"env":     builder.RunMode,
+				"server":  builder.ServerName,
-			"appname": builder.AppName,
+				"env":     builder.RunMode,
-		},
+				"appname": builder.AppName,
-		Help: "The statics info for remote http requests",
+			},
-	}, []string{"proto", "scheme", "method", "host", "path", "status", "duration", "isError"})
+			Help: "The statics info for remote http requests",
+		}, []string{"proto", "scheme", "method", "host", "path", "status", "isError"})
+
+		prometheus.MustRegister(summaryVec)
+	})
 
 	return func(ctx context.Context, req *httplib.BeegoHTTPRequest) (*http.Response, error) {
 		startTime := time.Now()
@@ -72,6 +79,6 @@ func (builder *FilterChainBuilder) report(startTime time.Time, endTime time.Time
 
 	dur := int(endTime.Sub(startTime) / time.Millisecond)
 
-	builder.summaryVec.WithLabelValues(proto, scheme, method, host, path,
+	summaryVec.WithLabelValues(proto, scheme, method, host, path,
-		strconv.Itoa(status), strconv.Itoa(dur), strconv.FormatBool(err == nil))
+		strconv.Itoa(status), strconv.FormatBool(err != nil)).Observe(float64(dur))
 }

client/orm/filter/prometheus/filter.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -33,24 +34,29 @@ import (
 // if we want to records the metrics of QuerySetter
 // actually we only records metrics of invoking "QueryTable" and "QueryTableWithCtx"
 type FilterChainBuilder struct {
-	summaryVec prometheus.ObserverVec
 	AppName    string
 	ServerName string
 	RunMode    string
 }
 
+var summaryVec prometheus.ObserverVec
+var initSummaryVec sync.Once
+
 func (builder *FilterChainBuilder) FilterChain(next orm.Filter) orm.Filter {
 
-	builder.summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+	initSummaryVec.Do(func() {
-		Name:      "beego",
+		summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Subsystem: "orm_operation",
+			Name:      "beego",
-		ConstLabels: map[string]string{
+			Subsystem: "orm_operation",
-			"server":  builder.ServerName,
+			ConstLabels: map[string]string{
-			"env":     builder.RunMode,
+				"server":  builder.ServerName,
-			"appname": builder.AppName,
+				"env":     builder.RunMode,
-		},
+				"appname": builder.AppName,
-		Help: "The statics info for orm operation",
+			},
-	}, []string{"method", "name", "duration", "insideTx", "txName"})
+			Help: "The statics info for orm operation",
+		}, []string{"method", "name", "insideTx", "txName"})
+		prometheus.MustRegister(summaryVec)
+	})
 
 	return func(ctx context.Context, inv *orm.Invocation) []interface{} {
 		startTime := time.Now()
@@ -74,12 +80,12 @@ func (builder *FilterChainBuilder) report(ctx context.Context, inv *orm.Invocati
 		builder.reportTxn(ctx, inv)
 		return
 	}
-	builder.summaryVec.WithLabelValues(inv.Method, inv.GetTableName(), strconv.Itoa(int(dur)),
+	summaryVec.WithLabelValues(inv.Method, inv.GetTableName(),
-		strconv.FormatBool(inv.InsideTx), inv.TxName)
+		strconv.FormatBool(inv.InsideTx), inv.TxName).Observe(float64(dur))
 }
 
 func (builder *FilterChainBuilder) reportTxn(ctx context.Context, inv *orm.Invocation) {
 	dur := time.Now().Sub(inv.TxStartTime) / time.Millisecond
-	builder.summaryVec.WithLabelValues(inv.Method, inv.TxName, strconv.Itoa(int(dur)),
+	summaryVec.WithLabelValues(inv.Method, inv.TxName,
-		strconv.FormatBool(inv.InsideTx), inv.TxName)
+		strconv.FormatBool(inv.InsideTx), inv.TxName).Observe(float64(dur))
 }

client/orm/filter/prometheus/filter_test.go

@@ -32,7 +32,7 @@ func TestFilterChainBuilder_FilterChain1(t *testing.T) {
 	builder := &FilterChainBuilder{}
 	filter := builder.FilterChain(next)
 
-	assert.NotNil(t, builder.summaryVec)
+	assert.NotNil(t, summaryVec)
 	assert.NotNil(t, filter)
 
 	inv := &orm.Invocation{}

go.mod

@@ -12,9 +12,9 @@ require (
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
-	github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d
+	github.com/couchbase/go-couchbase v0.0.0-20201216133707-c04035124b17
-	github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808 // indirect
+	github.com/couchbase/gomemcached v0.1.2-0.20201215185628-3bc3f73e68cb // indirect
-	github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a // indirect
+	github.com/couchbase/goutils v0.0.0-20201030094643-5e82bb967e67 // indirect
 	github.com/elastic/go-elasticsearch/v6 v6.8.5
 	github.com/elazarl/go-bindata-assetfs v1.0.0
 	github.com/go-kit/kit v0.9.0

go.sum

@@ -40,10 +40,16 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d h1:OMrhQqj1QCyDT2sxHCDjE+k8aMdn2ngTCGG7g4wrdLo=
 github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U=
+github.com/couchbase/go-couchbase v0.0.0-20201216133707-c04035124b17 h1:1ZELwRDUvpBpmgKSIUP6VMW1jIehzD0sCdWxRyejegw=
+github.com/couchbase/go-couchbase v0.0.0-20201216133707-c04035124b17/go.mod h1:+/bddYDxXsf9qt0xpDUtRR47A2GjaXmGGAqQ/k3GJ8A=
 github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808 h1:8s2l8TVUwMXl6tZMe3+hPCRJ25nQXiA3d1x622JtOqc=
 github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
+github.com/couchbase/gomemcached v0.1.2-0.20201215185628-3bc3f73e68cb h1:ZCFku0K/3Xvl7rXkGGM+ioT76Rxko8V9wDEWa0GFp14=
+github.com/couchbase/gomemcached v0.1.2-0.20201215185628-3bc3f73e68cb/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo=
 github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a h1:Y5XsLCEhtEI8qbD9RP3Qlv5FXdTDHxZM9UPUnMRgBp8=
 github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
+github.com/couchbase/goutils v0.0.0-20201030094643-5e82bb967e67 h1:NCqJ6fwen6YP0WlV/IyibaT0kPt3JEI1rA62V/UPKT4=
+github.com/couchbase/goutils v0.0.0-20201030094643-5e82bb967e67/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 h1:Lgdd/Qp96Qj8jqLpq2cI1I1X7BJnu06efS+XkhRoLUQ=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -257,6 +263,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrS
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -281,6 +288,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4=
 golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

server/web/filter/prometheus/filter.go

@@ -17,23 +17,49 @@ package prometheus
 import (
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/beego/beego/v2"
+	"github.com/beego/beego/v2/core/logs"
 	"github.com/beego/beego/v2/server/web"
 	"github.com/beego/beego/v2/server/web/context"
 )
 
+const unknownRouterPattern = "UnknownRouterPattern"
+
 // FilterChainBuilder is an extension point,
 // when we want to support some configuration,
 // please use this structure
 type FilterChainBuilder struct {
 }
 
+var summaryVec prometheus.ObserverVec
+var initSummaryVec sync.Once
+
 // FilterChain returns a FilterFunc. The filter will records some metrics
 func (builder *FilterChainBuilder) FilterChain(next web.FilterFunc) web.FilterFunc {
+
+	initSummaryVec.Do(func() {
+		summaryVec = builder.buildVec()
+		err := prometheus.Register(summaryVec)
+		if _, ok := err.(*prometheus.AlreadyRegisteredError); err != nil && !ok {
+			logs.Error("web module register prometheus vector failed, %+v", err)
+		}
+		registerBuildInfo()
+	})
+
+	return func(ctx *context.Context) {
+		startTime := time.Now()
+		next(ctx)
+		endTime := time.Now()
+		go report(endTime.Sub(startTime), ctx, summaryVec)
+	}
+}
+
+func (builder *FilterChainBuilder) buildVec() *prometheus.SummaryVec {
 	summaryVec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
 		Name:      "beego",
 		Subsystem: "http_request",
@@ -43,18 +69,8 @@ func (builder *FilterChainBuilder) FilterChain(next web.FilterFunc) web.FilterFu
 			"appname": web.BConfig.AppName,
 		},
 		Help: "The statics info for http request",
-	}, []string{"pattern", "method", "status", "duration"})
+	}, []string{"pattern", "method", "status"})
-
+	return summaryVec
-	prometheus.MustRegister(summaryVec)
-
-	registerBuildInfo()
-
-	return func(ctx *context.Context) {
-		startTime := time.Now()
-		next(ctx)
-		endTime := time.Now()
-		go report(endTime.Sub(startTime), ctx, summaryVec)
-	}
 }
 
 func registerBuildInfo() {
@@ -75,13 +91,17 @@ func registerBuildInfo() {
 		},
 	}, []string{})
 
-	prometheus.MustRegister(buildInfo)
+	_ = prometheus.Register(buildInfo)
 	buildInfo.WithLabelValues().Set(1)
 }
 
-func report(dur time.Duration, ctx *context.Context, vec *prometheus.SummaryVec) {
+func report(dur time.Duration, ctx *context.Context, vec prometheus.ObserverVec) {
 	status := ctx.Output.Status
-	ptn := ctx.Input.GetData("RouterPattern").(string)
+	ptnItf := ctx.Input.GetData("RouterPattern")
+	ptn := unknownRouterPattern
+	if ptnItf != nil {
+		ptn = ptnItf.(string)
+	}
 	ms := dur / time.Millisecond
-	vec.WithLabelValues(ptn, ctx.Input.Method(), strconv.Itoa(status), strconv.Itoa(int(ms))).Observe(float64(ms))
+	vec.WithLabelValues(ptn, ctx.Input.Method(), strconv.Itoa(status)).Observe(float64(ms))
 }

server/web/filter/prometheus/filter_test.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -38,3 +39,17 @@ func TestFilterChain(t *testing.T) {
 	filter(ctx)
 	assert.True(t, ctx.Input.GetData("invocation").(bool))
 }
+
+func TestFilterChainBuilder_report(t *testing.T) {
+
+	ctx := context.NewContext()
+	r, _ := http.NewRequest("GET", "/prometheus/user", nil)
+	w := httptest.NewRecorder()
+	ctx.Reset(w, r)
+	fb := &FilterChainBuilder{}
+	// without router info
+	report(time.Second, ctx, fb.buildVec())
+
+	ctx.Input.SetData("RouterPattern", "my-route")
+	report(time.Second, ctx, fb.buildVec())
+}

task/task.go

@@ -157,6 +157,9 @@ func (t *Task) GetSpec(context.Context) string {
 func (t *Task) GetStatus(context.Context) string {
 	var str string
 	for _, v := range t.Errlist {
+		if v == nil {
+			continue
+		}
 		str += v.t.String() + ":" + v.errinfo + "<br>"
 	}
 	return str

task/task_test.go

@@ -36,9 +36,11 @@ func TestParse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	assert.Equal(t, "0/30 * * * * *", tk.GetSpec(context.Background()))
 	m.AddTask("taska", tk)
 	m.StartTask()
 	time.Sleep(3 * time.Second)
+	assert.True(t, len(tk.GetStatus(context.Background())) == 0)
 	m.StopTask()
 }